2024-11-11 22:25:10,695 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 22:25:10,711 main DEBUG Took 0.013966 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 22:25:10,712 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 22:25:10,712 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 22:25:10,714 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 22:25:10,715 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,724 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 22:25:10,738 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,740 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,741 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,743 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,744 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,746 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,746 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,747 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,747 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,748 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,749 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,750 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,751 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,752 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,752 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,752 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,753 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,754 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:25:10,754 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,754 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 22:25:10,756 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:25:10,757 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 22:25:10,759 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 22:25:10,759 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 22:25:10,760 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 22:25:10,761 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 22:25:10,768 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 22:25:10,770 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 22:25:10,772 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 22:25:10,773 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 22:25:10,773 main DEBUG createAppenders(={Console}) 2024-11-11 22:25:10,774 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 22:25:10,774 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 22:25:10,774 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 22:25:10,775 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 22:25:10,775 main DEBUG OutputStream closed 2024-11-11 22:25:10,776 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 22:25:10,776 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 22:25:10,776 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 22:25:10,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 22:25:10,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 22:25:10,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 22:25:10,850 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 22:25:10,850 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 22:25:10,851 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 22:25:10,851 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 22:25:10,851 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 22:25:10,852 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 22:25:10,852 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 22:25:10,852 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 22:25:10,853 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 22:25:10,853 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 22:25:10,854 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 22:25:10,854 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 22:25:10,854 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 22:25:10,854 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 22:25:10,855 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 22:25:10,858 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 22:25:10,858 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 22:25:10,858 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 22:25:10,859 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T22:25:11,103 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af 2024-11-11 22:25:11,107 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 22:25:11,107 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T22:25:11,115 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-11T22:25:11,135 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T22:25:11,149 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/cluster_b777580f-683f-c355-fb6d-abef9c123800, deleteOnExit=true 2024-11-11T22:25:11,149 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T22:25:11,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/test.cache.data in system properties and HBase conf 2024-11-11T22:25:11,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T22:25:11,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/hadoop.log.dir in system properties and HBase conf 2024-11-11T22:25:11,152 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T22:25:11,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T22:25:11,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T22:25:11,241 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T22:25:11,332 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T22:25:11,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T22:25:11,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T22:25:11,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T22:25:11,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T22:25:11,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T22:25:11,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T22:25:11,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T22:25:11,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T22:25:11,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T22:25:11,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/nfs.dump.dir in system properties and HBase conf 2024-11-11T22:25:11,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/java.io.tmpdir in system properties and HBase conf 2024-11-11T22:25:11,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T22:25:11,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T22:25:11,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T22:25:13,456 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T22:25:13,527 INFO [Time-limited test {}] log.Log(170): Logging initialized @3679ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T22:25:13,605 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T22:25:13,677 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T22:25:13,707 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T22:25:13,708 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T22:25:13,710 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T22:25:13,726 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T22:25:13,729 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/hadoop.log.dir/,AVAILABLE} 2024-11-11T22:25:13,730 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T22:25:13,908 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/java.io.tmpdir/jetty-localhost-45979-hadoop-hdfs-3_4_1-tests_jar-_-any-685862834019727185/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T22:25:13,916 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:45979} 2024-11-11T22:25:13,917 INFO [Time-limited test {}] server.Server(415): Started @4069ms 2024-11-11T22:25:14,619 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T22:25:14,628 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T22:25:14,629 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T22:25:14,629 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T22:25:14,629 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T22:25:14,630 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/hadoop.log.dir/,AVAILABLE} 2024-11-11T22:25:14,631 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T22:25:14,733 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/java.io.tmpdir/jetty-localhost-35193-hadoop-hdfs-3_4_1-tests_jar-_-any-465847239492537951/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T22:25:14,734 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:35193} 2024-11-11T22:25:14,734 INFO [Time-limited test {}] server.Server(415): Started @4887ms 2024-11-11T22:25:14,788 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T22:25:15,864 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/cluster_b777580f-683f-c355-fb6d-abef9c123800/data/data2/current/BP-1035355522-172.17.0.3-1731363912033/current, will proceed with Du for space computation calculation, 2024-11-11T22:25:15,864 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/cluster_b777580f-683f-c355-fb6d-abef9c123800/data/data1/current/BP-1035355522-172.17.0.3-1731363912033/current, will proceed with Du for space computation calculation, 2024-11-11T22:25:15,893 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T22:25:15,938 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa52069d1816b568b with lease ID 0x4d27357a869f306: Processing first storage report for DS-a410b3c2-cc20-4f5b-add6-3ce8c276f359 from datanode DatanodeRegistration(127.0.0.1:34299, datanodeUuid=5d1d97ad-3b73-4a78-b4d0-810df608b925, infoPort=46419, infoSecurePort=0, ipcPort=43763, storageInfo=lv=-57;cid=testClusterID;nsid=1866122879;c=1731363912033) 2024-11-11T22:25:15,939 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa52069d1816b568b with lease ID 0x4d27357a869f306: from storage DS-a410b3c2-cc20-4f5b-add6-3ce8c276f359 node DatanodeRegistration(127.0.0.1:34299, datanodeUuid=5d1d97ad-3b73-4a78-b4d0-810df608b925, infoPort=46419, infoSecurePort=0, ipcPort=43763, storageInfo=lv=-57;cid=testClusterID;nsid=1866122879;c=1731363912033), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T22:25:15,939 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa52069d1816b568b with lease ID 0x4d27357a869f306: Processing first storage report for DS-f993722d-916c-44ad-98dd-e309dd5a46fb from datanode DatanodeRegistration(127.0.0.1:34299, datanodeUuid=5d1d97ad-3b73-4a78-b4d0-810df608b925, infoPort=46419, infoSecurePort=0, ipcPort=43763, storageInfo=lv=-57;cid=testClusterID;nsid=1866122879;c=1731363912033) 2024-11-11T22:25:15,940 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa52069d1816b568b with lease ID 0x4d27357a869f306: from storage DS-f993722d-916c-44ad-98dd-e309dd5a46fb node DatanodeRegistration(127.0.0.1:34299, datanodeUuid=5d1d97ad-3b73-4a78-b4d0-810df608b925, infoPort=46419, infoSecurePort=0, ipcPort=43763, storageInfo=lv=-57;cid=testClusterID;nsid=1866122879;c=1731363912033), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T22:25:16,035 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af 2024-11-11T22:25:16,096 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/cluster_b777580f-683f-c355-fb6d-abef9c123800/zookeeper_0, clientPort=55789, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/cluster_b777580f-683f-c355-fb6d-abef9c123800/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/cluster_b777580f-683f-c355-fb6d-abef9c123800/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T22:25:16,105 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55789 2024-11-11T22:25:16,114 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:25:16,116 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:25:17,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741825_1001 (size=7) 2024-11-11T22:25:17,426 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e with version=8 2024-11-11T22:25:17,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/hbase-staging 2024-11-11T22:25:17,517 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T22:25:17,719 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8d82b89dd27d:0 server-side Connection retries=45 2024-11-11T22:25:17,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T22:25:17,728 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T22:25:17,732 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T22:25:17,733 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T22:25:17,733 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T22:25:17,865 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T22:25:17,922 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T22:25:17,931 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T22:25:17,935 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T22:25:17,958 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 34148 (auto-detected) 2024-11-11T22:25:17,959 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-11T22:25:17,978 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40133 2024-11-11T22:25:17,997 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40133 connecting to ZooKeeper ensemble=127.0.0.1:55789 2024-11-11T22:25:18,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:401330x0, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T22:25:18,111 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40133-0x1012c00c4dd0000 connected 2024-11-11T22:25:18,208 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:25:18,210 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:25:18,220 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T22:25:18,224 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e, hbase.cluster.distributed=false 2024-11-11T22:25:18,245 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T22:25:18,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40133 2024-11-11T22:25:18,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40133 2024-11-11T22:25:18,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40133 2024-11-11T22:25:18,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40133 2024-11-11T22:25:18,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40133 2024-11-11T22:25:18,345 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8d82b89dd27d:0 server-side Connection retries=45 2024-11-11T22:25:18,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T22:25:18,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T22:25:18,347 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T22:25:18,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T22:25:18,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T22:25:18,350 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T22:25:18,352 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T22:25:18,353 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39613 2024-11-11T22:25:18,355 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39613 connecting to ZooKeeper ensemble=127.0.0.1:55789 2024-11-11T22:25:18,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:25:18,360 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:25:18,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396130x0, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T22:25:18,374 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:396130x0, quorum=127.0.0.1:55789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T22:25:18,374 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39613-0x1012c00c4dd0001 connected 2024-11-11T22:25:18,378 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T22:25:18,388 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T22:25:18,391 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T22:25:18,398 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T22:25:18,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39613 2024-11-11T22:25:18,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39613 2024-11-11T22:25:18,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39613 2024-11-11T22:25:18,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39613 2024-11-11T22:25:18,410 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39613 2024-11-11T22:25:18,429 DEBUG [M:0;8d82b89dd27d:40133 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8d82b89dd27d:40133 2024-11-11T22:25:18,430 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:18,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T22:25:18,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T22:25:18,450 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:18,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T22:25:18,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:18,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:18,480 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T22:25:18,481 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8d82b89dd27d,40133,1731363917560 from backup master directory 2024-11-11T22:25:18,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:18,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T22:25:18,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T22:25:18,490 WARN [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T22:25:18,490 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:18,492 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T22:25:18,493 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T22:25:18,546 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/hbase.id] with ID: cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe 2024-11-11T22:25:18,546 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/.tmp/hbase.id 2024-11-11T22:25:18,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741826_1002 (size=42) 2024-11-11T22:25:18,961 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/.tmp/hbase.id]:[hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/hbase.id] 2024-11-11T22:25:19,011 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:25:19,016 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T22:25:19,035 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-11T22:25:19,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:19,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:19,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741827_1003 (size=196) 2024-11-11T22:25:19,532 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:25:19,535 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T22:25:19,548 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:19,552 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T22:25:19,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741828_1004 (size=1189) 2024-11-11T22:25:20,006 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store 2024-11-11T22:25:20,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741829_1005 (size=34) 2024-11-11T22:25:20,430 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T22:25:20,433 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:25:20,434 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T22:25:20,434 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:25:20,434 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:25:20,436 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T22:25:20,436 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:25:20,436 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:25:20,437 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731363920434Disabling compacts and flushes for region at 1731363920434Disabling writes for close at 1731363920436 (+2 ms)Writing region close event to WAL at 1731363920436Closed at 1731363920436 2024-11-11T22:25:20,439 WARN [master/8d82b89dd27d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/.initializing 2024-11-11T22:25:20,439 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/WALs/8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:20,448 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T22:25:20,462 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8d82b89dd27d%2C40133%2C1731363917560, suffix=, logDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/WALs/8d82b89dd27d,40133,1731363917560, archiveDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/oldWALs, maxLogs=10 2024-11-11T22:25:20,484 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/WALs/8d82b89dd27d,40133,1731363917560/8d82b89dd27d%2C40133%2C1731363917560.1731363920466, exclude list is [], retry=0 2024-11-11T22:25:20,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34299,DS-a410b3c2-cc20-4f5b-add6-3ce8c276f359,DISK] 2024-11-11T22:25:20,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T22:25:20,539 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/WALs/8d82b89dd27d,40133,1731363917560/8d82b89dd27d%2C40133%2C1731363917560.1731363920466 2024-11-11T22:25:20,540 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46419:46419)] 2024-11-11T22:25:20,540 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:25:20,541 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:25:20,544 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,545 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T22:25:20,615 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:20,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:25:20,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T22:25:20,623 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:20,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:20,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T22:25:20,630 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:20,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:20,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T22:25:20,636 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:20,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:20,638 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,642 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,643 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,652 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,653 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,658 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T22:25:20,664 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:25:20,669 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:25:20,670 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72427562, jitterRate=0.07925477623939514}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T22:25:20,677 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731363920561Initializing all the Stores at 1731363920564 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363920564Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363920565 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363920565Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363920565Cleaning up temporary data from old regions at 1731363920653 (+88 ms)Region opened successfully at 1731363920677 (+24 ms) 2024-11-11T22:25:20,678 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T22:25:20,715 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e5d3043, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8d82b89dd27d/172.17.0.3:0 2024-11-11T22:25:20,753 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T22:25:20,765 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T22:25:20,765 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T22:25:20,768 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T22:25:20,770 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T22:25:20,775 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-11T22:25:20,775 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T22:25:20,812 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T22:25:20,825 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T22:25:20,878 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T22:25:20,883 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T22:25:20,887 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T22:25:20,899 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T22:25:20,904 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T22:25:20,910 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T22:25:20,983 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T22:25:20,986 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T22:25:20,994 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T22:25:21,016 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T22:25:21,025 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T22:25:21,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T22:25:21,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T22:25:21,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:21,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:21,040 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8d82b89dd27d,40133,1731363917560, sessionid=0x1012c00c4dd0000, setting cluster-up flag (Was=false) 2024-11-11T22:25:21,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:21,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:21,099 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T22:25:21,102 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:21,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:21,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:21,152 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T22:25:21,154 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:21,162 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T22:25:21,217 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(746): ClusterId : cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe 2024-11-11T22:25:21,220 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T22:25:21,236 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T22:25:21,236 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T22:25:21,244 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T22:25:21,249 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T22:25:21,249 DEBUG [RS:0;8d82b89dd27d:39613 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a2ada5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8d82b89dd27d/172.17.0.3:0 2024-11-11T22:25:21,255 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T22:25:21,264 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T22:25:21,271 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8d82b89dd27d:39613 2024-11-11T22:25:21,270 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8d82b89dd27d,40133,1731363917560 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T22:25:21,275 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T22:25:21,276 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T22:25:21,276 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T22:25:21,279 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(2659): reportForDuty to master=8d82b89dd27d,40133,1731363917560 with port=39613, startcode=1731363918313 2024-11-11T22:25:21,281 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8d82b89dd27d:0, corePoolSize=5, maxPoolSize=5 2024-11-11T22:25:21,281 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8d82b89dd27d:0, corePoolSize=5, maxPoolSize=5 2024-11-11T22:25:21,281 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8d82b89dd27d:0, corePoolSize=5, maxPoolSize=5 2024-11-11T22:25:21,282 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8d82b89dd27d:0, corePoolSize=5, maxPoolSize=5 2024-11-11T22:25:21,282 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8d82b89dd27d:0, corePoolSize=10, maxPoolSize=10 2024-11-11T22:25:21,284 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,285 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8d82b89dd27d:0, corePoolSize=2, maxPoolSize=2 2024-11-11T22:25:21,285 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,293 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731363951293 2024-11-11T22:25:21,294 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T22:25:21,295 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T22:25:21,296 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T22:25:21,297 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T22:25:21,298 DEBUG [RS:0;8d82b89dd27d:39613 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T22:25:21,301 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T22:25:21,302 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T22:25:21,303 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T22:25:21,303 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T22:25:21,305 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:21,305 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T22:25:21,310 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,315 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T22:25:21,316 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T22:25:21,317 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T22:25:21,320 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T22:25:21,321 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T22:25:21,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741831_1007 (size=1321) 2024-11-11T22:25:21,328 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.large.0-1731363921322,5,FailOnTimeoutGroup] 2024-11-11T22:25:21,332 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.small.0-1731363921328,5,FailOnTimeoutGroup] 2024-11-11T22:25:21,332 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,333 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T22:25:21,334 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,334 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,357 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47247, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T22:25:21,366 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40133 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:21,370 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40133 {}] master.ServerManager(517): Registering regionserver=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:21,391 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e 2024-11-11T22:25:21,391 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46449 2024-11-11T22:25:21,391 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T22:25:21,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T22:25:21,405 DEBUG [RS:0;8d82b89dd27d:39613 {}] zookeeper.ZKUtil(111): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:21,406 WARN [RS:0;8d82b89dd27d:39613 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T22:25:21,406 INFO [RS:0;8d82b89dd27d:39613 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T22:25:21,406 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/WALs/8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:21,408 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8d82b89dd27d,39613,1731363918313] 2024-11-11T22:25:21,441 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T22:25:21,463 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T22:25:21,469 INFO [RS:0;8d82b89dd27d:39613 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T22:25:21,470 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,471 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T22:25:21,477 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T22:25:21,479 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,479 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,479 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,480 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,480 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,480 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,480 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8d82b89dd27d:0, corePoolSize=2, maxPoolSize=2 2024-11-11T22:25:21,480 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,480 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,481 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,481 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,481 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,481 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:25:21,481 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8d82b89dd27d:0, corePoolSize=3, maxPoolSize=3 2024-11-11T22:25:21,482 DEBUG [RS:0;8d82b89dd27d:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0, corePoolSize=3, maxPoolSize=3 2024-11-11T22:25:21,484 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,484 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,485 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,485 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,485 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,485 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,39613,1731363918313-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T22:25:21,516 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T22:25:21,519 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,39613,1731363918313-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,519 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,520 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.Replication(171): 8d82b89dd27d,39613,1731363918313 started 2024-11-11T22:25:21,547 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:21,548 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1482): Serving as 8d82b89dd27d,39613,1731363918313, RpcServer on 8d82b89dd27d/172.17.0.3:39613, sessionid=0x1012c00c4dd0001 2024-11-11T22:25:21,549 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T22:25:21,549 DEBUG [RS:0;8d82b89dd27d:39613 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:21,549 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8d82b89dd27d,39613,1731363918313' 2024-11-11T22:25:21,550 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T22:25:21,551 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T22:25:21,552 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T22:25:21,552 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T22:25:21,552 DEBUG [RS:0;8d82b89dd27d:39613 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:21,552 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8d82b89dd27d,39613,1731363918313' 2024-11-11T22:25:21,553 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T22:25:21,554 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T22:25:21,554 DEBUG [RS:0;8d82b89dd27d:39613 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T22:25:21,555 INFO [RS:0;8d82b89dd27d:39613 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T22:25:21,555 INFO [RS:0;8d82b89dd27d:39613 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T22:25:21,661 INFO [RS:0;8d82b89dd27d:39613 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T22:25:21,666 INFO [RS:0;8d82b89dd27d:39613 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8d82b89dd27d%2C39613%2C1731363918313, suffix=, logDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/WALs/8d82b89dd27d,39613,1731363918313, archiveDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/oldWALs, maxLogs=32 2024-11-11T22:25:21,690 DEBUG [RS:0;8d82b89dd27d:39613 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/WALs/8d82b89dd27d,39613,1731363918313/8d82b89dd27d%2C39613%2C1731363918313.1731363921669, exclude list is [], retry=0 2024-11-11T22:25:21,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34299,DS-a410b3c2-cc20-4f5b-add6-3ce8c276f359,DISK] 2024-11-11T22:25:21,700 INFO [RS:0;8d82b89dd27d:39613 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/WALs/8d82b89dd27d,39613,1731363918313/8d82b89dd27d%2C39613%2C1731363918313.1731363921669 2024-11-11T22:25:21,701 DEBUG [RS:0;8d82b89dd27d:39613 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46419:46419)] 2024-11-11T22:25:21,726 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T22:25:21,726 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e 2024-11-11T22:25:21,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741833_1009 (size=32) 2024-11-11T22:25:22,142 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:25:22,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T22:25:22,147 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T22:25:22,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:22,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:25:22,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T22:25:22,152 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T22:25:22,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:22,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:25:22,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T22:25:22,157 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T22:25:22,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:22,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:25:22,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T22:25:22,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T22:25:22,164 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:22,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:25:22,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T22:25:22,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740 2024-11-11T22:25:22,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740 2024-11-11T22:25:22,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T22:25:22,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T22:25:22,175 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:25:22,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T22:25:22,183 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:25:22,185 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73484000, jitterRate=0.09499692916870117}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:25:22,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731363922142Initializing all the Stores at 1731363922144 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363922144Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363922144Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363922144Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363922144Cleaning up temporary data from old regions at 1731363922174 (+30 ms)Region opened successfully at 1731363922188 (+14 ms) 2024-11-11T22:25:22,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T22:25:22,188 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T22:25:22,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T22:25:22,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T22:25:22,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T22:25:22,190 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T22:25:22,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731363922188Disabling compacts and flushes for region at 1731363922188Disabling writes for close at 1731363922189 (+1 ms)Writing region close event to WAL at 1731363922190 (+1 ms)Closed at 1731363922190 2024-11-11T22:25:22,195 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T22:25:22,196 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T22:25:22,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T22:25:22,215 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T22:25:22,219 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T22:25:22,372 DEBUG [8d82b89dd27d:40133 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T22:25:22,384 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:22,390 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8d82b89dd27d,39613,1731363918313, state=OPENING 2024-11-11T22:25:22,535 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T22:25:22,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:22,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:25:22,554 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T22:25:22,554 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T22:25:22,557 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T22:25:22,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:25:22,734 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T22:25:22,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33345, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T22:25:22,751 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T22:25:22,751 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T22:25:22,751 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T22:25:22,755 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8d82b89dd27d%2C39613%2C1731363918313.meta, suffix=.meta, logDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/WALs/8d82b89dd27d,39613,1731363918313, archiveDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/oldWALs, maxLogs=32 2024-11-11T22:25:22,772 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/WALs/8d82b89dd27d,39613,1731363918313/8d82b89dd27d%2C39613%2C1731363918313.meta.1731363922757.meta, exclude list is [], retry=0 2024-11-11T22:25:22,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34299,DS-a410b3c2-cc20-4f5b-add6-3ce8c276f359,DISK] 2024-11-11T22:25:22,780 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/WALs/8d82b89dd27d,39613,1731363918313/8d82b89dd27d%2C39613%2C1731363918313.meta.1731363922757.meta 2024-11-11T22:25:22,781 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46419:46419)] 2024-11-11T22:25:22,781 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:25:22,783 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T22:25:22,785 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T22:25:22,790 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T22:25:22,794 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T22:25:22,795 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:25:22,795 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T22:25:22,795 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T22:25:22,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T22:25:22,800 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T22:25:22,800 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:22,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:25:22,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T22:25:22,803 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T22:25:22,803 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:22,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:25:22,805 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T22:25:22,806 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T22:25:22,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:22,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:25:22,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T22:25:22,809 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T22:25:22,809 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:22,810 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:25:22,811 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T22:25:22,812 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740 2024-11-11T22:25:22,816 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740 2024-11-11T22:25:22,818 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T22:25:22,818 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T22:25:22,819 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:25:22,822 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T22:25:22,824 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71453947, jitterRate=0.06474678218364716}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:25:22,824 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T22:25:22,825 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731363922796Writing region info on filesystem at 1731363922796Initializing all the Stores at 1731363922797 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363922798 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363922798Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363922798Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363922798Cleaning up temporary data from old regions at 1731363922819 (+21 ms)Running coprocessor post-open hooks at 1731363922824 (+5 ms)Region opened successfully at 1731363922825 (+1 ms) 2024-11-11T22:25:22,849 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731363922725 2024-11-11T22:25:22,862 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T22:25:22,863 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T22:25:22,865 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:22,868 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8d82b89dd27d,39613,1731363918313, state=OPEN 2024-11-11T22:25:22,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T22:25:22,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T22:25:22,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T22:25:22,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T22:25:22,927 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:22,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T22:25:22,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8d82b89dd27d,39613,1731363918313 in 367 msec 2024-11-11T22:25:22,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T22:25:22,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 731 msec 2024-11-11T22:25:22,943 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T22:25:22,943 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T22:25:22,961 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:22,962 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:22,981 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:22,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54221, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:23,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.8070 sec 2024-11-11T22:25:23,005 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731363923005, completionTime=-1 2024-11-11T22:25:23,007 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T22:25:23,008 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T22:25:23,069 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-11T22:25:23,069 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731363983069 2024-11-11T22:25:23,070 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731364043069 2024-11-11T22:25:23,070 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 61 msec 2024-11-11T22:25:23,072 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,40133,1731363917560-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:23,072 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,40133,1731363917560-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:23,073 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,40133,1731363917560-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:23,075 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8d82b89dd27d:40133, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:23,075 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:23,082 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:23,087 DEBUG [master/8d82b89dd27d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T22:25:23,109 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.619sec 2024-11-11T22:25:23,110 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T22:25:23,111 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T22:25:23,112 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T22:25:23,112 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T22:25:23,112 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T22:25:23,113 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,40133,1731363917560-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T22:25:23,113 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,40133,1731363917560-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T22:25:23,122 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T22:25:23,123 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T22:25:23,123 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,40133,1731363917560-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:25:23,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:23,129 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T22:25:23,129 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T22:25:23,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:23,133 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:23,141 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:23,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:23,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:23,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:23,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:23,157 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:23,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:23,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:23,202 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=7032 2024-11-11T22:25:23,211 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:25:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-11-11T22:25:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:25:23,224 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:25:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:25:23,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:25:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:25:23,245 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:25:23,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T22:25:23,248 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:23,250 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:25:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:25:23,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741835_1011 (size=959) 2024-11-11T22:25:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:25:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:25:23,676 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e 2024-11-11T22:25:23,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741836_1012 (size=53) 2024-11-11T22:25:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:25:24,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:25:24,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 765de8aa1edd426d207e09298c569dc6, disabling compactions & flushes 2024-11-11T22:25:24,087 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:24,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:24,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. after waiting 0 ms 2024-11-11T22:25:24,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:24,087 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:24,088 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 765de8aa1edd426d207e09298c569dc6: Waiting for close lock at 1731363924087Disabling compacts and flushes for region at 1731363924087Disabling writes for close at 1731363924087Writing region close event to WAL at 1731363924087Closed at 1731363924087 2024-11-11T22:25:24,090 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:25:24,096 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731363924090"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731363924090"}]},"ts":"1731363924090"} 2024-11-11T22:25:24,101 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:25:24,103 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:25:24,107 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363924104"}]},"ts":"1731363924104"} 2024-11-11T22:25:24,111 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:25:24,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=765de8aa1edd426d207e09298c569dc6, ASSIGN}] 2024-11-11T22:25:24,116 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=765de8aa1edd426d207e09298c569dc6, ASSIGN 2024-11-11T22:25:24,118 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=765de8aa1edd426d207e09298c569dc6, ASSIGN; state=OFFLINE, location=8d82b89dd27d,39613,1731363918313; forceNewPlan=false, retain=false 2024-11-11T22:25:24,269 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=765de8aa1edd426d207e09298c569dc6, regionState=OPENING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:24,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=765de8aa1edd426d207e09298c569dc6, ASSIGN because future has completed 2024-11-11T22:25:24,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:25:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:25:24,439 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:24,440 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:25:24,440 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,440 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:25:24,441 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,441 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,443 INFO [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,447 INFO [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 765de8aa1edd426d207e09298c569dc6 columnFamilyName A 2024-11-11T22:25:24,447 DEBUG [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:24,448 INFO [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] regionserver.HStore(327): Store=765de8aa1edd426d207e09298c569dc6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:24,449 INFO [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,452 INFO [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 765de8aa1edd426d207e09298c569dc6 columnFamilyName B 2024-11-11T22:25:24,452 DEBUG [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:24,453 INFO [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] regionserver.HStore(327): Store=765de8aa1edd426d207e09298c569dc6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:24,454 INFO [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,457 INFO [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 765de8aa1edd426d207e09298c569dc6 columnFamilyName C 2024-11-11T22:25:24,457 DEBUG [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:24,460 INFO [StoreOpener-765de8aa1edd426d207e09298c569dc6-1 {}] regionserver.HStore(327): Store=765de8aa1edd426d207e09298c569dc6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:24,461 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,463 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,465 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,467 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,468 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,469 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:25:24,472 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,476 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:25:24,477 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 765de8aa1edd426d207e09298c569dc6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71643393, jitterRate=0.06756974756717682}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:25:24,477 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:24,478 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 765de8aa1edd426d207e09298c569dc6: Running coprocessor pre-open hook at 1731363924441Writing region info on filesystem at 1731363924441Initializing all the Stores at 1731363924443 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363924443Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363924443Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363924443Cleaning up temporary data from old regions at 1731363924468 (+25 ms)Running coprocessor post-open hooks at 1731363924477 (+9 ms)Region opened successfully at 1731363924478 (+1 ms) 2024-11-11T22:25:24,480 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., pid=6, masterSystemTime=1731363924429 2024-11-11T22:25:24,484 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:24,484 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:24,486 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=765de8aa1edd426d207e09298c569dc6, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:24,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:25:24,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T22:25:24,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 in 219 msec 2024-11-11T22:25:24,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T22:25:24,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=765de8aa1edd426d207e09298c569dc6, ASSIGN in 387 msec 2024-11-11T22:25:24,507 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:25:24,508 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363924508"}]},"ts":"1731363924508"} 2024-11-11T22:25:24,512 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:25:24,514 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:25:24,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2780 sec 2024-11-11T22:25:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:25:25,405 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:25,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:25,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:25,417 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec19887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:25,420 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621eb0cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,422 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:25,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:25,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:25,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:25,429 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:25:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@49cb0867 2024-11-11T22:25:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:25:25,436 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:25:25,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:25,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:25,442 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180c72f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:25,446 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355546b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:25,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:25,450 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a9376a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:25,454 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:25,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4c56da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:25,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:25,457 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:25,461 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:25,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:25,464 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:25,468 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b27b3a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:25,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:25,471 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d96214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:25,474 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f776e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:25,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:25,477 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:25,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6e4493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:25,480 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1455c669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:25,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:25,483 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:25,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:25,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555d2363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:25,485 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f9c971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:25,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:25,489 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e23982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:25,491 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94111c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:25,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:25,494 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe80a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:25,496 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b322e04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:25,502 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:25,502 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:25,502 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:25,504 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:25,504 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:25,504 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:25,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:25,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:25,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:25,508 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:25,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:25,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:25,510 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:25,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:25,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:25,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:25:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:25:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:25:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:25:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:25:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:25:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:25:25,527 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:25,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:25,534 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:25:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:25,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:25:25,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T22:25:25,562 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:25,565 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:25,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:25:25,616 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:25,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:25:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:25:25,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/30d2d97281ef4e70a22de3e2b3674109 is 50, key is test_row_0/A:col10/1731363925615/Put/seqid=0 2024-11-11T22:25:25,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:25,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:25,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:25,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:25,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:25,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:25,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741837_1013 (size=14341) 2024-11-11T22:25:25,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/30d2d97281ef4e70a22de3e2b3674109 2024-11-11T22:25:25,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:25,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363985788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363985781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:25,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363985795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:25,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363985802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:25,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363985803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:25:25,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:25,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/c3be75bc59814b4ba3d3c501d8e71c06 is 50, key is test_row_0/B:col10/1731363925615/Put/seqid=0 2024-11-11T22:25:25,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:25,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:25,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363985956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:25,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:25,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363985956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741838_1014 (size=12001) 2024-11-11T22:25:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:25,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/c3be75bc59814b4ba3d3c501d8e71c06 2024-11-11T22:25:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363985956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:25,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363985957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:25,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363985955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/490fe132312744d3ad89d4842967577e is 50, key is test_row_0/C:col10/1731363925615/Put/seqid=0 2024-11-11T22:25:26,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741839_1015 (size=12001) 2024-11-11T22:25:26,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/490fe132312744d3ad89d4842967577e 2024-11-11T22:25:26,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/30d2d97281ef4e70a22de3e2b3674109 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/30d2d97281ef4e70a22de3e2b3674109 2024-11-11T22:25:26,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/30d2d97281ef4e70a22de3e2b3674109, entries=200, sequenceid=13, filesize=14.0 K 2024-11-11T22:25:26,079 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T22:25:26,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/c3be75bc59814b4ba3d3c501d8e71c06 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c3be75bc59814b4ba3d3c501d8e71c06 2024-11-11T22:25:26,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:26,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c3be75bc59814b4ba3d3c501d8e71c06, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T22:25:26,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/490fe132312744d3ad89d4842967577e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/490fe132312744d3ad89d4842967577e 2024-11-11T22:25:26,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:26,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/490fe132312744d3ad89d4842967577e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T22:25:26,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 765de8aa1edd426d207e09298c569dc6 in 524ms, sequenceid=13, compaction requested=false 2024-11-11T22:25:26,148 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T22:25:26,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:26,173 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:26,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T22:25:26,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/29d5f51210ba4d6f8dfac29a0327dd44 is 50, key is test_row_0/A:col10/1731363925794/Put/seqid=0 2024-11-11T22:25:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:25:26,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363986192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741840_1016 (size=14341) 2024-11-11T22:25:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363986217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/29d5f51210ba4d6f8dfac29a0327dd44 2024-11-11T22:25:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/274920d3f8894c89b60a6859d2057f29 is 50, key is test_row_0/B:col10/1731363925794/Put/seqid=0 2024-11-11T22:25:26,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363986234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363986235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363986242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741841_1017 (size=12001) 2024-11-11T22:25:26,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/274920d3f8894c89b60a6859d2057f29 2024-11-11T22:25:26,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:26,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:26,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/a1d98d1298734b699aad6738fc4ab1c0 is 50, key is test_row_0/C:col10/1731363925794/Put/seqid=0 2024-11-11T22:25:26,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363986335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741842_1018 (size=12001) 2024-11-11T22:25:26,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363986345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363986375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363986376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363986376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363986544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363986574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363986594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363986594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363986605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:25:26,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/a1d98d1298734b699aad6738fc4ab1c0 2024-11-11T22:25:26,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:26,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:26,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/29d5f51210ba4d6f8dfac29a0327dd44 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/29d5f51210ba4d6f8dfac29a0327dd44 2024-11-11T22:25:26,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/29d5f51210ba4d6f8dfac29a0327dd44, entries=200, sequenceid=39, filesize=14.0 K 2024-11-11T22:25:26,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/274920d3f8894c89b60a6859d2057f29 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/274920d3f8894c89b60a6859d2057f29 2024-11-11T22:25:26,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/274920d3f8894c89b60a6859d2057f29, entries=150, sequenceid=39, filesize=11.7 K 2024-11-11T22:25:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/a1d98d1298734b699aad6738fc4ab1c0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1d98d1298734b699aad6738fc4ab1c0 2024-11-11T22:25:26,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1d98d1298734b699aad6738fc4ab1c0, entries=150, sequenceid=39, filesize=11.7 K 2024-11-11T22:25:26,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 765de8aa1edd426d207e09298c569dc6 in 669ms, sequenceid=39, compaction requested=false 2024-11-11T22:25:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:26,873 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:26,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:25:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:26,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/c5af2a140d024c8593716d5f71a83fbf is 50, key is test_row_0/A:col10/1731363926856/Put/seqid=0 2024-11-11T22:25:26,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:26,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:26,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:26,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741843_1019 (size=14341) 2024-11-11T22:25:26,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363986983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:26,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363986987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:26,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:26,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363986992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363986992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363986995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:27,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:27,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:27,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:27,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:27,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:27,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363987105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363987105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363987106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363987113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363987113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:27,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:27,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:27,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:27,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:27,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363987314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363987323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363987324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363987324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363987324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/c5af2a140d024c8593716d5f71a83fbf 2024-11-11T22:25:27,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/9ea3f7b271e24008b208f07e4797558e is 50, key is test_row_0/B:col10/1731363926856/Put/seqid=0 2024-11-11T22:25:27,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741844_1020 (size=12001) 2024-11-11T22:25:27,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/9ea3f7b271e24008b208f07e4797558e 2024-11-11T22:25:27,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:27,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:27,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/f0af9870b7ef4caf8d68d31c8242a819 is 50, key is test_row_0/C:col10/1731363926856/Put/seqid=0 2024-11-11T22:25:27,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741845_1021 (size=12001) 2024-11-11T22:25:27,433 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T22:25:27,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/f0af9870b7ef4caf8d68d31c8242a819 2024-11-11T22:25:27,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/c5af2a140d024c8593716d5f71a83fbf as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c5af2a140d024c8593716d5f71a83fbf 2024-11-11T22:25:27,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c5af2a140d024c8593716d5f71a83fbf, entries=200, sequenceid=51, filesize=14.0 K 2024-11-11T22:25:27,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/9ea3f7b271e24008b208f07e4797558e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/9ea3f7b271e24008b208f07e4797558e 2024-11-11T22:25:27,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/9ea3f7b271e24008b208f07e4797558e, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T22:25:27,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/f0af9870b7ef4caf8d68d31c8242a819 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f0af9870b7ef4caf8d68d31c8242a819 2024-11-11T22:25:27,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f0af9870b7ef4caf8d68d31c8242a819, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T22:25:27,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 765de8aa1edd426d207e09298c569dc6 in 631ms, sequenceid=51, compaction requested=true 2024-11-11T22:25:27,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:27,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:27,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:27,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:27,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:27,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:27,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:25:27,514 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:27,514 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:27,518 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:27,520 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:27,521 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:27,521 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:27,521 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:27,521 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:27,522 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c3be75bc59814b4ba3d3c501d8e71c06, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/274920d3f8894c89b60a6859d2057f29, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/9ea3f7b271e24008b208f07e4797558e] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=35.2 K 2024-11-11T22:25:27,523 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/30d2d97281ef4e70a22de3e2b3674109, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/29d5f51210ba4d6f8dfac29a0327dd44, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c5af2a140d024c8593716d5f71a83fbf] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=42.0 K 2024-11-11T22:25:27,524 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting c3be75bc59814b4ba3d3c501d8e71c06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363925594 2024-11-11T22:25:27,525 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 274920d3f8894c89b60a6859d2057f29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731363925794 2024-11-11T22:25:27,525 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30d2d97281ef4e70a22de3e2b3674109, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363925594 2024-11-11T22:25:27,526 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d5f51210ba4d6f8dfac29a0327dd44, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731363925786 2024-11-11T22:25:27,527 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5af2a140d024c8593716d5f71a83fbf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731363926211 2024-11-11T22:25:27,532 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ea3f7b271e24008b208f07e4797558e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731363926211 2024-11-11T22:25:27,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:25:27,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:27,562 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:25:27,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/cba64166e777475a9176acc9ad483969 is 50, key is test_row_0/A:col10/1731363926991/Put/seqid=0 2024-11-11T22:25:27,617 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#11 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:27,618 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/ca348bcabe7b446a8cf5a7c41fc61b44 is 50, key is test_row_0/B:col10/1731363926856/Put/seqid=0 2024-11-11T22:25:27,623 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#10 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:27,624 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/ed80ea7c5d4043de9414bfb99d6fe219 is 50, key is test_row_0/A:col10/1731363926856/Put/seqid=0 2024-11-11T22:25:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:27,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:27,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741846_1022 (size=12001) 2024-11-11T22:25:27,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741847_1023 (size=12104) 2024-11-11T22:25:27,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741848_1024 (size=12104) 2024-11-11T22:25:27,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363987666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363987683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363987687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363987696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363987699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:25:27,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363987804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363987814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363987824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363987824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:27,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363987824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363988013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363988033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363988044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363988044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363988045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,076 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/cba64166e777475a9176acc9ad483969 2024-11-11T22:25:28,104 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/ca348bcabe7b446a8cf5a7c41fc61b44 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ca348bcabe7b446a8cf5a7c41fc61b44 2024-11-11T22:25:28,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/be6d2a2d40794c97b5c2dfe9a49cadc7 is 50, key is test_row_0/B:col10/1731363926991/Put/seqid=0 2024-11-11T22:25:28,122 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/ed80ea7c5d4043de9414bfb99d6fe219 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/ed80ea7c5d4043de9414bfb99d6fe219 2024-11-11T22:25:28,140 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into ca348bcabe7b446a8cf5a7c41fc61b44(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:28,140 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:28,144 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=13, startTime=1731363927512; duration=0sec 2024-11-11T22:25:28,145 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:28,146 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:28,146 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:28,149 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:28,150 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:28,150 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:28,157 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/490fe132312744d3ad89d4842967577e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1d98d1298734b699aad6738fc4ab1c0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f0af9870b7ef4caf8d68d31c8242a819] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=35.2 K 2024-11-11T22:25:28,158 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 490fe132312744d3ad89d4842967577e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363925594 2024-11-11T22:25:28,160 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into ed80ea7c5d4043de9414bfb99d6fe219(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:28,160 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:28,160 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=13, startTime=1731363927506; duration=0sec 2024-11-11T22:25:28,160 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:28,161 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:28,161 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a1d98d1298734b699aad6738fc4ab1c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731363925794 2024-11-11T22:25:28,162 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f0af9870b7ef4caf8d68d31c8242a819, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731363926211 2024-11-11T22:25:28,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741849_1025 (size=12001) 2024-11-11T22:25:28,166 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/be6d2a2d40794c97b5c2dfe9a49cadc7 2024-11-11T22:25:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/13b44a140e1e4b8e8ab6d5094af7ca22 is 50, key is test_row_0/C:col10/1731363926991/Put/seqid=0 2024-11-11T22:25:28,215 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#14 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:28,216 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/e25d41164771421989019d14d92bc7e9 is 50, key is test_row_0/C:col10/1731363926856/Put/seqid=0 2024-11-11T22:25:28,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741850_1026 (size=12001) 2024-11-11T22:25:28,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741851_1027 (size=12104) 2024-11-11T22:25:28,282 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/e25d41164771421989019d14d92bc7e9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e25d41164771421989019d14d92bc7e9 2024-11-11T22:25:28,300 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into e25d41164771421989019d14d92bc7e9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:28,300 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:28,300 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=13, startTime=1731363927512; duration=0sec 2024-11-11T22:25:28,301 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:28,301 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:28,318 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:25:28,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363988324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363988356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363988363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363988364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363988365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,468 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T22:25:28,469 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T22:25:28,488 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T22:25:28,488 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T22:25:28,488 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T22:25:28,488 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T22:25:28,490 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T22:25:28,490 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T22:25:28,638 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/13b44a140e1e4b8e8ab6d5094af7ca22 2024-11-11T22:25:28,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/cba64166e777475a9176acc9ad483969 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/cba64166e777475a9176acc9ad483969 2024-11-11T22:25:28,667 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/cba64166e777475a9176acc9ad483969, entries=150, sequenceid=75, filesize=11.7 K 2024-11-11T22:25:28,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/be6d2a2d40794c97b5c2dfe9a49cadc7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/be6d2a2d40794c97b5c2dfe9a49cadc7 2024-11-11T22:25:28,686 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/be6d2a2d40794c97b5c2dfe9a49cadc7, entries=150, sequenceid=75, filesize=11.7 K 2024-11-11T22:25:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/13b44a140e1e4b8e8ab6d5094af7ca22 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/13b44a140e1e4b8e8ab6d5094af7ca22 2024-11-11T22:25:28,729 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/13b44a140e1e4b8e8ab6d5094af7ca22, entries=150, sequenceid=75, filesize=11.7 K 2024-11-11T22:25:28,731 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 765de8aa1edd426d207e09298c569dc6 in 1169ms, sequenceid=75, compaction requested=false 2024-11-11T22:25:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T22:25:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T22:25:28,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T22:25:28,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1680 sec 2024-11-11T22:25:28,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.1890 sec 2024-11-11T22:25:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:28,848 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:28,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T22:25:28,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/05bcdf107cbf4c9d91932e50a884923e is 50, key is test_row_0/A:col10/1731363927682/Put/seqid=0 2024-11-11T22:25:28,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741852_1028 (size=14341) 2024-11-11T22:25:28,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363988939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363988940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363988942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363988947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363988947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363989054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363989054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363989054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363989074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363989074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363989264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363989264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363989264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/05bcdf107cbf4c9d91932e50a884923e 2024-11-11T22:25:29,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/e9a4bd2628fb4307808c41c20083d8f6 is 50, key is test_row_0/B:col10/1731363927682/Put/seqid=0 2024-11-11T22:25:29,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363989304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363989307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741853_1029 (size=12001) 2024-11-11T22:25:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363989575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363989575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363989575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363989613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363989624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:25:29,734 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:29,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T22:25:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:25:29,738 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:29,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:29,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:29,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/e9a4bd2628fb4307808c41c20083d8f6 2024-11-11T22:25:29,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9865bff02fd0499ca179673162be26ea is 50, key is test_row_0/C:col10/1731363927682/Put/seqid=0 2024-11-11T22:25:29,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741854_1030 (size=12001) 2024-11-11T22:25:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:25:29,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T22:25:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:29,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:30,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T22:25:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:30,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:25:30,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363990094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:30,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363990094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:30,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363990099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:30,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363990134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:30,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363990143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:30,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T22:25:30,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:30,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:30,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:30,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:30,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:30,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9865bff02fd0499ca179673162be26ea 2024-11-11T22:25:30,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/05bcdf107cbf4c9d91932e50a884923e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/05bcdf107cbf4c9d91932e50a884923e 2024-11-11T22:25:30,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/05bcdf107cbf4c9d91932e50a884923e, entries=200, sequenceid=91, filesize=14.0 K 2024-11-11T22:25:30,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/e9a4bd2628fb4307808c41c20083d8f6 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e9a4bd2628fb4307808c41c20083d8f6 2024-11-11T22:25:30,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e9a4bd2628fb4307808c41c20083d8f6, entries=150, sequenceid=91, filesize=11.7 K 2024-11-11T22:25:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9865bff02fd0499ca179673162be26ea as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9865bff02fd0499ca179673162be26ea 2024-11-11T22:25:30,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9865bff02fd0499ca179673162be26ea, entries=150, sequenceid=91, filesize=11.7 K 2024-11-11T22:25:30,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 765de8aa1edd426d207e09298c569dc6 in 1459ms, sequenceid=91, compaction requested=true 2024-11-11T22:25:30,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:30,308 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:30,308 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:30,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:30,311 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:30,311 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:30,311 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:30,312 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/ed80ea7c5d4043de9414bfb99d6fe219, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/cba64166e777475a9176acc9ad483969, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/05bcdf107cbf4c9d91932e50a884923e] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=37.5 K 2024-11-11T22:25:30,314 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:30,314 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:30,315 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:30,315 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ca348bcabe7b446a8cf5a7c41fc61b44, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/be6d2a2d40794c97b5c2dfe9a49cadc7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e9a4bd2628fb4307808c41c20083d8f6] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=35.3 K 2024-11-11T22:25:30,316 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed80ea7c5d4043de9414bfb99d6fe219, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731363926211 2024-11-11T22:25:30,317 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting ca348bcabe7b446a8cf5a7c41fc61b44, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731363926211 2024-11-11T22:25:30,317 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting cba64166e777475a9176acc9ad483969, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731363926946 2024-11-11T22:25:30,318 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting be6d2a2d40794c97b5c2dfe9a49cadc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731363926946 2024-11-11T22:25:30,321 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting e9a4bd2628fb4307808c41c20083d8f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731363927682 2024-11-11T22:25:30,321 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05bcdf107cbf4c9d91932e50a884923e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731363927682 2024-11-11T22:25:30,354 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:30,355 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/8e8c9e661906423980d1120b1105acaf is 50, key is test_row_0/A:col10/1731363927682/Put/seqid=0 2024-11-11T22:25:30,358 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:30,359 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/cab2d8da292a4920bc1301f89e2dbeb8 is 50, key is test_row_0/B:col10/1731363927682/Put/seqid=0 2024-11-11T22:25:30,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T22:25:30,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:30,368 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:25:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:25:30,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741855_1031 (size=12207) 2024-11-11T22:25:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e589c5556a1349559380cee995023238 is 50, key is test_row_0/A:col10/1731363928942/Put/seqid=0 2024-11-11T22:25:30,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741856_1032 (size=12207) 2024-11-11T22:25:30,409 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/8e8c9e661906423980d1120b1105acaf as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8e8c9e661906423980d1120b1105acaf 2024-11-11T22:25:30,423 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/cab2d8da292a4920bc1301f89e2dbeb8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cab2d8da292a4920bc1301f89e2dbeb8 2024-11-11T22:25:30,425 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into 8e8c9e661906423980d1120b1105acaf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:30,426 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:30,426 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=13, startTime=1731363930307; duration=0sec 2024-11-11T22:25:30,426 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:30,426 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:30,426 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:30,434 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:30,434 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:30,435 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:30,435 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e25d41164771421989019d14d92bc7e9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/13b44a140e1e4b8e8ab6d5094af7ca22, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9865bff02fd0499ca179673162be26ea] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=35.3 K 2024-11-11T22:25:30,436 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting e25d41164771421989019d14d92bc7e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731363926211 2024-11-11T22:25:30,437 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13b44a140e1e4b8e8ab6d5094af7ca22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731363926946 2024-11-11T22:25:30,438 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9865bff02fd0499ca179673162be26ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731363927682 2024-11-11T22:25:30,449 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into cab2d8da292a4920bc1301f89e2dbeb8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:30,450 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:30,450 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=13, startTime=1731363930308; duration=0sec 2024-11-11T22:25:30,450 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:30,450 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:30,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741857_1033 (size=12001) 2024-11-11T22:25:30,474 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:30,475 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/173b164761434d88b5d586b76e675157 is 50, key is test_row_0/C:col10/1731363927682/Put/seqid=0 2024-11-11T22:25:30,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741858_1034 (size=12207) 2024-11-11T22:25:30,858 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e589c5556a1349559380cee995023238 2024-11-11T22:25:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/c4b9166c8a6843a6b35dc664f6758209 is 50, key is test_row_0/B:col10/1731363928942/Put/seqid=0 2024-11-11T22:25:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:25:30,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741859_1035 (size=12001) 2024-11-11T22:25:30,914 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/c4b9166c8a6843a6b35dc664f6758209 2024-11-11T22:25:30,934 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/173b164761434d88b5d586b76e675157 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/173b164761434d88b5d586b76e675157 2024-11-11T22:25:30,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/4364401ff7ad41d7bad970ef0a004c42 is 50, key is test_row_0/C:col10/1731363928942/Put/seqid=0 2024-11-11T22:25:30,961 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into 173b164761434d88b5d586b76e675157(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:30,961 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:30,961 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=13, startTime=1731363930308; duration=0sec 2024-11-11T22:25:30,961 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:30,961 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:30,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741860_1036 (size=12001) 2024-11-11T22:25:30,968 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/4364401ff7ad41d7bad970ef0a004c42 2024-11-11T22:25:30,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e589c5556a1349559380cee995023238 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e589c5556a1349559380cee995023238 2024-11-11T22:25:30,991 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e589c5556a1349559380cee995023238, entries=150, sequenceid=115, filesize=11.7 K 2024-11-11T22:25:30,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/c4b9166c8a6843a6b35dc664f6758209 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c4b9166c8a6843a6b35dc664f6758209 2024-11-11T22:25:31,013 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c4b9166c8a6843a6b35dc664f6758209, entries=150, sequenceid=115, filesize=11.7 K 2024-11-11T22:25:31,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/4364401ff7ad41d7bad970ef0a004c42 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4364401ff7ad41d7bad970ef0a004c42 2024-11-11T22:25:31,038 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4364401ff7ad41d7bad970ef0a004c42, entries=150, sequenceid=115, filesize=11.7 K 2024-11-11T22:25:31,040 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 765de8aa1edd426d207e09298c569dc6 in 672ms, sequenceid=115, compaction requested=false 2024-11-11T22:25:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T22:25:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T22:25:31,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T22:25:31,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3040 sec 2024-11-11T22:25:31,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3140 sec 2024-11-11T22:25:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:31,153 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:31,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:25:31,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/7702e3d74c1b43b686aff98c156a806e is 50, key is test_row_0/A:col10/1731363931147/Put/seqid=0 2024-11-11T22:25:31,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741861_1037 (size=12051) 2024-11-11T22:25:31,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/7702e3d74c1b43b686aff98c156a806e 2024-11-11T22:25:31,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/fe41aa28213a456f9a483788a93bc2a1 is 50, key is test_row_0/B:col10/1731363931147/Put/seqid=0 2024-11-11T22:25:31,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741862_1038 (size=12051) 2024-11-11T22:25:31,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363991277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363991278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363991280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363991282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363991288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363991394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363991394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363991403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363991404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363991404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363991604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363991604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363991613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363991614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363991617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/fe41aa28213a456f9a483788a93bc2a1 2024-11-11T22:25:31,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/c82edde4ba404dd6ae83ee80a8b3ee84 is 50, key is test_row_0/C:col10/1731363931147/Put/seqid=0 2024-11-11T22:25:31,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741863_1039 (size=12051) 2024-11-11T22:25:31,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/c82edde4ba404dd6ae83ee80a8b3ee84 2024-11-11T22:25:31,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/7702e3d74c1b43b686aff98c156a806e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/7702e3d74c1b43b686aff98c156a806e 2024-11-11T22:25:31,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/7702e3d74c1b43b686aff98c156a806e, entries=150, sequenceid=129, filesize=11.8 K 2024-11-11T22:25:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/fe41aa28213a456f9a483788a93bc2a1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fe41aa28213a456f9a483788a93bc2a1 2024-11-11T22:25:31,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fe41aa28213a456f9a483788a93bc2a1, entries=150, sequenceid=129, filesize=11.8 K 2024-11-11T22:25:31,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/c82edde4ba404dd6ae83ee80a8b3ee84 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/c82edde4ba404dd6ae83ee80a8b3ee84 2024-11-11T22:25:31,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/c82edde4ba404dd6ae83ee80a8b3ee84, entries=150, sequenceid=129, filesize=11.8 K 2024-11-11T22:25:31,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 765de8aa1edd426d207e09298c569dc6 in 580ms, sequenceid=129, compaction requested=true 2024-11-11T22:25:31,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:31,734 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:31,734 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:31,736 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:31,736 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:31,736 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:31,736 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:31,736 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:31,736 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:31,736 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cab2d8da292a4920bc1301f89e2dbeb8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c4b9166c8a6843a6b35dc664f6758209, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fe41aa28213a456f9a483788a93bc2a1] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=35.4 K 2024-11-11T22:25:31,736 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8e8c9e661906423980d1120b1105acaf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e589c5556a1349559380cee995023238, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/7702e3d74c1b43b686aff98c156a806e] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=35.4 K 2024-11-11T22:25:31,737 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting cab2d8da292a4920bc1301f89e2dbeb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731363927682 2024-11-11T22:25:31,738 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e8c9e661906423980d1120b1105acaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731363927682 2024-11-11T22:25:31,738 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting c4b9166c8a6843a6b35dc664f6758209, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731363928932 2024-11-11T22:25:31,738 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting e589c5556a1349559380cee995023238, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731363928932 2024-11-11T22:25:31,739 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting fe41aa28213a456f9a483788a93bc2a1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731363931138 2024-11-11T22:25:31,739 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7702e3d74c1b43b686aff98c156a806e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731363931138 2024-11-11T22:25:31,763 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:31,764 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/f49e885949e5497187afa9dbce4c935e is 50, key is test_row_0/B:col10/1731363931147/Put/seqid=0 2024-11-11T22:25:31,779 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:31,780 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/c65dfb7fa36d41d9a966bd173b61473e is 50, key is test_row_0/A:col10/1731363931147/Put/seqid=0 2024-11-11T22:25:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741864_1040 (size=12359) 2024-11-11T22:25:31,851 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/f49e885949e5497187afa9dbce4c935e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f49e885949e5497187afa9dbce4c935e 2024-11-11T22:25:31,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741865_1041 (size=12359) 2024-11-11T22:25:31,870 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/c65dfb7fa36d41d9a966bd173b61473e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c65dfb7fa36d41d9a966bd173b61473e 2024-11-11T22:25:31,870 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into f49e885949e5497187afa9dbce4c935e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:31,870 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:31,870 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=13, startTime=1731363931734; duration=0sec 2024-11-11T22:25:31,870 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:31,871 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:31,871 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:31,878 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:31,879 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:31,879 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:31,879 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/173b164761434d88b5d586b76e675157, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4364401ff7ad41d7bad970ef0a004c42, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/c82edde4ba404dd6ae83ee80a8b3ee84] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=35.4 K 2024-11-11T22:25:31,880 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 173b164761434d88b5d586b76e675157, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731363927682 2024-11-11T22:25:31,882 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4364401ff7ad41d7bad970ef0a004c42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731363928932 2024-11-11T22:25:31,883 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting c82edde4ba404dd6ae83ee80a8b3ee84, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731363931138 2024-11-11T22:25:31,892 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into c65dfb7fa36d41d9a966bd173b61473e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:31,892 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:31,892 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=13, startTime=1731363931734; duration=0sec 2024-11-11T22:25:31,893 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:25:31,893 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:31,894 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:31,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:31,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T22:25:31,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T22:25:31,899 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:31,902 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:31,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:31,926 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:31,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T22:25:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:31,928 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#29 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:31,929 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/389e1a10b7ad47b7a0da3ec103703a2f is 50, key is test_row_0/C:col10/1731363931147/Put/seqid=0 2024-11-11T22:25:31,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/eac2c030dcce4c728db3ba2ea724a5c8 is 50, key is test_row_0/A:col10/1731363931283/Put/seqid=0 2024-11-11T22:25:31,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363991943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363991942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741866_1042 (size=12359) 2024-11-11T22:25:31,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363991958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363991962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363991969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:31,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:31,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:31,991 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/389e1a10b7ad47b7a0da3ec103703a2f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/389e1a10b7ad47b7a0da3ec103703a2f 2024-11-11T22:25:32,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741867_1043 (size=12151) 2024-11-11T22:25:32,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/eac2c030dcce4c728db3ba2ea724a5c8 2024-11-11T22:25:32,002 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into 389e1a10b7ad47b7a0da3ec103703a2f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:32,002 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:32,003 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=13, startTime=1731363931734; duration=0sec 2024-11-11T22:25:32,003 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:32,003 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T22:25:32,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/034dad3fe2ff4d15bc700bd7b575510c is 50, key is test_row_0/B:col10/1731363931283/Put/seqid=0 2024-11-11T22:25:32,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T22:25:32,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741868_1044 (size=12151) 2024-11-11T22:25:32,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363992073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363992073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/034dad3fe2ff4d15bc700bd7b575510c 2024-11-11T22:25:32,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363992083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363992084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363992094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/8ff6db10d086472b936c6bafebf33dff is 50, key is test_row_0/C:col10/1731363931283/Put/seqid=0 2024-11-11T22:25:32,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741869_1045 (size=12151) 2024-11-11T22:25:32,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/8ff6db10d086472b936c6bafebf33dff 2024-11-11T22:25:32,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/eac2c030dcce4c728db3ba2ea724a5c8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/eac2c030dcce4c728db3ba2ea724a5c8 2024-11-11T22:25:32,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/eac2c030dcce4c728db3ba2ea724a5c8, entries=150, sequenceid=158, filesize=11.9 K 2024-11-11T22:25:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/034dad3fe2ff4d15bc700bd7b575510c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/034dad3fe2ff4d15bc700bd7b575510c 2024-11-11T22:25:32,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/034dad3fe2ff4d15bc700bd7b575510c, entries=150, sequenceid=158, filesize=11.9 K 2024-11-11T22:25:32,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/8ff6db10d086472b936c6bafebf33dff as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/8ff6db10d086472b936c6bafebf33dff 2024-11-11T22:25:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T22:25:32,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T22:25:32,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:32,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/8ff6db10d086472b936c6bafebf33dff, entries=150, sequenceid=158, filesize=11.9 K 2024-11-11T22:25:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 765de8aa1edd426d207e09298c569dc6 in 300ms, sequenceid=158, compaction requested=false 2024-11-11T22:25:32,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:32,291 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:32,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:25:32,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/41beb201d8f4408eaa00be0ca53fdb9e is 50, key is test_row_0/A:col10/1731363931946/Put/seqid=0 2024-11-11T22:25:32,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741870_1046 (size=14541) 2024-11-11T22:25:32,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/41beb201d8f4408eaa00be0ca53fdb9e 2024-11-11T22:25:32,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363992345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363992349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363992355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/f28e12cebed34aa99ded2de0150f9aab is 50, key is test_row_0/B:col10/1731363931946/Put/seqid=0 2024-11-11T22:25:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363992357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363992359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T22:25:32,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:32,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741871_1047 (size=12151) 2024-11-11T22:25:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/f28e12cebed34aa99ded2de0150f9aab 2024-11-11T22:25:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/69aca1bddbce4be68838a3d11534e351 is 50, key is test_row_0/C:col10/1731363931946/Put/seqid=0 2024-11-11T22:25:32,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741872_1048 (size=12151) 2024-11-11T22:25:32,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/69aca1bddbce4be68838a3d11534e351 2024-11-11T22:25:32,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/41beb201d8f4408eaa00be0ca53fdb9e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/41beb201d8f4408eaa00be0ca53fdb9e 2024-11-11T22:25:32,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/41beb201d8f4408eaa00be0ca53fdb9e, entries=200, sequenceid=170, filesize=14.2 K 2024-11-11T22:25:32,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/f28e12cebed34aa99ded2de0150f9aab as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f28e12cebed34aa99ded2de0150f9aab 2024-11-11T22:25:32,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f28e12cebed34aa99ded2de0150f9aab, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T22:25:32,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/69aca1bddbce4be68838a3d11534e351 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/69aca1bddbce4be68838a3d11534e351 2024-11-11T22:25:32,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363992464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363992464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/69aca1bddbce4be68838a3d11534e351, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T22:25:32,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 765de8aa1edd426d207e09298c569dc6 in 185ms, sequenceid=170, compaction requested=true 2024-11-11T22:25:32,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:32,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:32,477 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:32,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:32,477 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:32,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:32,479 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:32,479 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:32,479 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,479 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:32,479 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:32,479 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,479 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c65dfb7fa36d41d9a966bd173b61473e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/eac2c030dcce4c728db3ba2ea724a5c8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/41beb201d8f4408eaa00be0ca53fdb9e] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=38.1 K 2024-11-11T22:25:32,479 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f49e885949e5497187afa9dbce4c935e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/034dad3fe2ff4d15bc700bd7b575510c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f28e12cebed34aa99ded2de0150f9aab] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=35.8 K 2024-11-11T22:25:32,480 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting c65dfb7fa36d41d9a966bd173b61473e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731363931138 2024-11-11T22:25:32,480 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f49e885949e5497187afa9dbce4c935e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731363931138 2024-11-11T22:25:32,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:32,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:32,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:32,481 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 034dad3fe2ff4d15bc700bd7b575510c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731363931283 2024-11-11T22:25:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:32,481 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting eac2c030dcce4c728db3ba2ea724a5c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731363931283 2024-11-11T22:25:32,482 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f28e12cebed34aa99ded2de0150f9aab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363931946 2024-11-11T22:25:32,482 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41beb201d8f4408eaa00be0ca53fdb9e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363931946 2024-11-11T22:25:32,488 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:32,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-11T22:25:32,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363992503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/1395214b2ac347d8a325b1ac56c06007 is 50, key is test_row_0/A:col10/1731363932481/Put/seqid=0 2024-11-11T22:25:32,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363992504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363992504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,515 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:32,516 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/eea258ecdf6f40ee94246667640aa32c is 50, key is test_row_0/B:col10/1731363931946/Put/seqid=0 2024-11-11T22:25:32,517 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T22:25:32,525 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/97d9c46de7644111959a58748f7e8f19 is 50, key is test_row_0/A:col10/1731363931946/Put/seqid=0 2024-11-11T22:25:32,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T22:25:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741875_1051 (size=12561) 2024-11-11T22:25:32,558 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/97d9c46de7644111959a58748f7e8f19 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/97d9c46de7644111959a58748f7e8f19 2024-11-11T22:25:32,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741874_1050 (size=12561) 2024-11-11T22:25:32,581 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into 97d9c46de7644111959a58748f7e8f19(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:32,581 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:32,581 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=13, startTime=1731363932477; duration=0sec 2024-11-11T22:25:32,581 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:32,581 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:32,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741873_1049 (size=16931) 2024-11-11T22:25:32,583 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:32,583 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/eea258ecdf6f40ee94246667640aa32c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/eea258ecdf6f40ee94246667640aa32c 2024-11-11T22:25:32,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/1395214b2ac347d8a325b1ac56c06007 2024-11-11T22:25:32,589 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:32,589 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:32,589 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,590 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/389e1a10b7ad47b7a0da3ec103703a2f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/8ff6db10d086472b936c6bafebf33dff, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/69aca1bddbce4be68838a3d11534e351] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=35.8 K 2024-11-11T22:25:32,591 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 389e1a10b7ad47b7a0da3ec103703a2f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731363931138 2024-11-11T22:25:32,592 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff6db10d086472b936c6bafebf33dff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731363931283 2024-11-11T22:25:32,594 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69aca1bddbce4be68838a3d11534e351, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363931946 2024-11-11T22:25:32,602 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into eea258ecdf6f40ee94246667640aa32c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:32,602 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:32,602 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=13, startTime=1731363932477; duration=0sec 2024-11-11T22:25:32,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/c1da72d648244434a753e1ebf7628ce4 is 50, key is test_row_0/B:col10/1731363932481/Put/seqid=0 2024-11-11T22:25:32,603 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:32,603 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:32,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363992614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363992623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,627 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:32,627 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/a1a9c1ec18d84014b70a5b1d3e05434f is 50, key is test_row_0/C:col10/1731363931946/Put/seqid=0 2024-11-11T22:25:32,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363992632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741876_1052 (size=12151) 2024-11-11T22:25:32,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/c1da72d648244434a753e1ebf7628ce4 2024-11-11T22:25:32,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741877_1053 (size=12561) 2024-11-11T22:25:32,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363992674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363992674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/875bcfbd894d43d4a478f434525c96d6 is 50, key is test_row_0/C:col10/1731363932481/Put/seqid=0 2024-11-11T22:25:32,687 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/a1a9c1ec18d84014b70a5b1d3e05434f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1a9c1ec18d84014b70a5b1d3e05434f 2024-11-11T22:25:32,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T22:25:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,699 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into a1a9c1ec18d84014b70a5b1d3e05434f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:32,699 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:32,699 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=13, startTime=1731363932480; duration=0sec 2024-11-11T22:25:32,699 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:32,699 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741878_1054 (size=12151) 2024-11-11T22:25:32,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363992824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363992833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363992844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T22:25:32,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:32,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:32,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363992984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:32,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363992984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T22:25:33,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:33,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:33,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:33,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:33,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T22:25:33,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/875bcfbd894d43d4a478f434525c96d6 2024-11-11T22:25:33,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363993143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363993143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/1395214b2ac347d8a325b1ac56c06007 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/1395214b2ac347d8a325b1ac56c06007 2024-11-11T22:25:33,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363993154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T22:25:33,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:33,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:33,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/1395214b2ac347d8a325b1ac56c06007, entries=250, sequenceid=199, filesize=16.5 K 2024-11-11T22:25:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/c1da72d648244434a753e1ebf7628ce4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c1da72d648244434a753e1ebf7628ce4 2024-11-11T22:25:33,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c1da72d648244434a753e1ebf7628ce4, entries=150, sequenceid=199, filesize=11.9 K 2024-11-11T22:25:33,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/875bcfbd894d43d4a478f434525c96d6 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/875bcfbd894d43d4a478f434525c96d6 2024-11-11T22:25:33,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/875bcfbd894d43d4a478f434525c96d6, entries=150, sequenceid=199, filesize=11.9 K 2024-11-11T22:25:33,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 765de8aa1edd426d207e09298c569dc6 in 728ms, sequenceid=199, compaction requested=false 2024-11-11T22:25:33,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:33,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T22:25:33,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:33,317 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-11T22:25:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/185aec4118c847d8825fc5338d72af8f is 50, key is test_row_0/A:col10/1731363932495/Put/seqid=0 2024-11-11T22:25:33,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741879_1055 (size=9757) 2024-11-11T22:25:33,344 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/185aec4118c847d8825fc5338d72af8f 2024-11-11T22:25:33,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/a000d67d37194504904c61ac655d901c is 50, key is test_row_0/B:col10/1731363932495/Put/seqid=0 2024-11-11T22:25:33,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741880_1056 (size=9757) 2024-11-11T22:25:33,386 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/a000d67d37194504904c61ac655d901c 2024-11-11T22:25:33,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/d9411f2a0f1a424a98b58c6988db7e86 is 50, key is test_row_0/C:col10/1731363932495/Put/seqid=0 2024-11-11T22:25:33,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741881_1057 (size=9757) 2024-11-11T22:25:33,432 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/d9411f2a0f1a424a98b58c6988db7e86 2024-11-11T22:25:33,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/185aec4118c847d8825fc5338d72af8f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/185aec4118c847d8825fc5338d72af8f 2024-11-11T22:25:33,469 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/185aec4118c847d8825fc5338d72af8f, entries=100, sequenceid=209, filesize=9.5 K 2024-11-11T22:25:33,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/a000d67d37194504904c61ac655d901c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/a000d67d37194504904c61ac655d901c 2024-11-11T22:25:33,488 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/a000d67d37194504904c61ac655d901c, entries=100, sequenceid=209, filesize=9.5 K 2024-11-11T22:25:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/d9411f2a0f1a424a98b58c6988db7e86 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d9411f2a0f1a424a98b58c6988db7e86 2024-11-11T22:25:33,504 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d9411f2a0f1a424a98b58c6988db7e86, entries=100, sequenceid=209, filesize=9.5 K 2024-11-11T22:25:33,505 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 765de8aa1edd426d207e09298c569dc6 in 188ms, sequenceid=209, compaction requested=true 2024-11-11T22:25:33,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:33,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:33,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T22:25:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T22:25:33,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T22:25:33,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6060 sec 2024-11-11T22:25:33,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.6190 sec 2024-11-11T22:25:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:33,527 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:33,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:25:33,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/79e5e9c94e824d148d394324041ba70d is 50, key is test_row_0/A:col10/1731363933509/Put/seqid=0 2024-11-11T22:25:33,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741882_1058 (size=14541) 2024-11-11T22:25:33,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/79e5e9c94e824d148d394324041ba70d 2024-11-11T22:25:33,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/fd2b044e719b4094afedebb3c20408a3 is 50, key is test_row_0/B:col10/1731363933509/Put/seqid=0 2024-11-11T22:25:33,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741883_1059 (size=12151) 2024-11-11T22:25:33,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363993621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363993625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363993653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363993653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363993663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363993733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363993733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363993943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363993943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:33,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/fd2b044e719b4094afedebb3c20408a3 2024-11-11T22:25:33,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/b6f05fcc430748ab8b716fcd0a495272 is 50, key is test_row_0/C:col10/1731363933509/Put/seqid=0 2024-11-11T22:25:34,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741884_1060 (size=12151) 2024-11-11T22:25:34,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/b6f05fcc430748ab8b716fcd0a495272 2024-11-11T22:25:34,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/79e5e9c94e824d148d394324041ba70d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/79e5e9c94e824d148d394324041ba70d 2024-11-11T22:25:34,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/79e5e9c94e824d148d394324041ba70d, entries=200, sequenceid=220, filesize=14.2 K 2024-11-11T22:25:34,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/fd2b044e719b4094afedebb3c20408a3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fd2b044e719b4094afedebb3c20408a3 2024-11-11T22:25:34,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fd2b044e719b4094afedebb3c20408a3, entries=150, sequenceid=220, filesize=11.9 K 2024-11-11T22:25:34,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/b6f05fcc430748ab8b716fcd0a495272 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b6f05fcc430748ab8b716fcd0a495272 2024-11-11T22:25:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T22:25:34,054 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:34,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T22:25:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T22:25:34,059 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:34,060 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:34,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:34,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b6f05fcc430748ab8b716fcd0a495272, entries=150, sequenceid=220, filesize=11.9 K 2024-11-11T22:25:34,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 765de8aa1edd426d207e09298c569dc6 in 552ms, sequenceid=220, compaction requested=true 2024-11-11T22:25:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:34,080 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:25:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:34,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:34,080 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:25:34,083 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:25:34,083 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:34,083 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:34,083 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/eea258ecdf6f40ee94246667640aa32c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c1da72d648244434a753e1ebf7628ce4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/a000d67d37194504904c61ac655d901c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fd2b044e719b4094afedebb3c20408a3] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=45.5 K 2024-11-11T22:25:34,084 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53790 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:25:34,084 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:34,084 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:34,084 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/97d9c46de7644111959a58748f7e8f19, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/1395214b2ac347d8a325b1ac56c06007, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/185aec4118c847d8825fc5338d72af8f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/79e5e9c94e824d148d394324041ba70d] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=52.5 K 2024-11-11T22:25:34,085 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting eea258ecdf6f40ee94246667640aa32c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363931946 2024-11-11T22:25:34,086 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97d9c46de7644111959a58748f7e8f19, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363931946 2024-11-11T22:25:34,086 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting c1da72d648244434a753e1ebf7628ce4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731363932349 2024-11-11T22:25:34,087 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1395214b2ac347d8a325b1ac56c06007, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731363932349 2024-11-11T22:25:34,087 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a000d67d37194504904c61ac655d901c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731363932495 2024-11-11T22:25:34,087 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185aec4118c847d8825fc5338d72af8f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731363932495 2024-11-11T22:25:34,087 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting fd2b044e719b4094afedebb3c20408a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731363933509 2024-11-11T22:25:34,088 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79e5e9c94e824d148d394324041ba70d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731363933509 2024-11-11T22:25:34,109 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:34,110 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/f21d8dffd38c4555a1e67aadc7636dac is 50, key is test_row_0/A:col10/1731363933509/Put/seqid=0 2024-11-11T22:25:34,119 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:34,120 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/d3f2c215b07e4c9d9e4bec61e8c31e96 is 50, key is test_row_0/B:col10/1731363933509/Put/seqid=0 2024-11-11T22:25:34,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741885_1061 (size=12697) 2024-11-11T22:25:34,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741886_1062 (size=12697) 2024-11-11T22:25:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T22:25:34,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T22:25:34,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:34,215 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:25:34,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/6cea76a3a74a4f2ab996fb95ac294620 is 50, key is test_row_0/A:col10/1731363933622/Put/seqid=0 2024-11-11T22:25:34,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741887_1063 (size=12151) 2024-11-11T22:25:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:34,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:34,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:34,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:34,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363994287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363994290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T22:25:34,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363994403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:34,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363994404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:34,551 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/f21d8dffd38c4555a1e67aadc7636dac as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/f21d8dffd38c4555a1e67aadc7636dac 2024-11-11T22:25:34,562 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/d3f2c215b07e4c9d9e4bec61e8c31e96 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/d3f2c215b07e4c9d9e4bec61e8c31e96 2024-11-11T22:25:34,565 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into f21d8dffd38c4555a1e67aadc7636dac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:34,565 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:34,565 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=12, startTime=1731363934080; duration=0sec 2024-11-11T22:25:34,565 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:34,565 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:34,566 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:25:34,567 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:25:34,567 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:34,568 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:34,568 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1a9c1ec18d84014b70a5b1d3e05434f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/875bcfbd894d43d4a478f434525c96d6, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d9411f2a0f1a424a98b58c6988db7e86, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b6f05fcc430748ab8b716fcd0a495272] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=45.5 K 2024-11-11T22:25:34,569 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1a9c1ec18d84014b70a5b1d3e05434f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363931946 2024-11-11T22:25:34,569 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 875bcfbd894d43d4a478f434525c96d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731363932349 2024-11-11T22:25:34,570 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9411f2a0f1a424a98b58c6988db7e86, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731363932495 2024-11-11T22:25:34,571 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6f05fcc430748ab8b716fcd0a495272, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731363933509 2024-11-11T22:25:34,573 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into d3f2c215b07e4c9d9e4bec61e8c31e96(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:34,574 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:34,574 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=12, startTime=1731363934080; duration=0sec 2024-11-11T22:25:34,574 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:34,574 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:34,588 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:34,589 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/af7b526f44cf4757948f93b1fa43b14b is 50, key is test_row_0/C:col10/1731363933509/Put/seqid=0 2024-11-11T22:25:34,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741888_1064 (size=12697) 2024-11-11T22:25:34,610 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/af7b526f44cf4757948f93b1fa43b14b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/af7b526f44cf4757948f93b1fa43b14b 2024-11-11T22:25:34,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363994613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:34,619 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into af7b526f44cf4757948f93b1fa43b14b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:34,619 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:34,619 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=12, startTime=1731363934080; duration=0sec 2024-11-11T22:25:34,619 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:34,619 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363994623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:34,651 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/6cea76a3a74a4f2ab996fb95ac294620 2024-11-11T22:25:34,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363994664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363994664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:34,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/00356badc91d4b55b1ca74726b2fe7f7 is 50, key is test_row_0/B:col10/1731363933622/Put/seqid=0 2024-11-11T22:25:34,672 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:34,673 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:34,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:34,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363994674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:34,676 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T22:25:34,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741889_1065 (size=12151) 2024-11-11T22:25:34,705 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/00356badc91d4b55b1ca74726b2fe7f7 2024-11-11T22:25:34,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9a7cd18c13974585acacbd1659df86d7 is 50, key is test_row_0/C:col10/1731363933622/Put/seqid=0 2024-11-11T22:25:34,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741890_1066 (size=12151) 2024-11-11T22:25:34,739 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9a7cd18c13974585acacbd1659df86d7 2024-11-11T22:25:34,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/6cea76a3a74a4f2ab996fb95ac294620 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/6cea76a3a74a4f2ab996fb95ac294620 2024-11-11T22:25:34,756 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/6cea76a3a74a4f2ab996fb95ac294620, entries=150, sequenceid=245, filesize=11.9 K 2024-11-11T22:25:34,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/00356badc91d4b55b1ca74726b2fe7f7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/00356badc91d4b55b1ca74726b2fe7f7 2024-11-11T22:25:34,764 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/00356badc91d4b55b1ca74726b2fe7f7, entries=150, sequenceid=245, filesize=11.9 K 2024-11-11T22:25:34,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9a7cd18c13974585acacbd1659df86d7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9a7cd18c13974585acacbd1659df86d7 2024-11-11T22:25:34,773 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9a7cd18c13974585acacbd1659df86d7, entries=150, sequenceid=245, filesize=11.9 K 2024-11-11T22:25:34,774 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 765de8aa1edd426d207e09298c569dc6 in 559ms, sequenceid=245, compaction requested=false 2024-11-11T22:25:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T22:25:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T22:25:34,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T22:25:34,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 716 msec 2024-11-11T22:25:34,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 725 msec 2024-11-11T22:25:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:34,925 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:34,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:25:34,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e7a840e1a0594912865974b7936146ef is 50, key is test_row_0/A:col10/1731363934923/Put/seqid=0 2024-11-11T22:25:34,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741891_1067 (size=14591) 2024-11-11T22:25:34,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e7a840e1a0594912865974b7936146ef 2024-11-11T22:25:34,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/92725a4904b244b8a7eec64a39080778 is 50, key is test_row_0/B:col10/1731363934923/Put/seqid=0 2024-11-11T22:25:34,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741892_1068 (size=12201) 2024-11-11T22:25:35,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363995026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:35,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:35,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363995030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:35,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:35,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363995133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:35,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363995143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T22:25:35,193 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:35,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T22:25:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:25:35,196 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:35,197 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:35,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:25:35,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363995343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:35,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T22:25:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363995354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:35,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/92725a4904b244b8a7eec64a39080778 2024-11-11T22:25:35,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/e42c623298224ce3bd50a71636d2ace3 is 50, key is test_row_0/C:col10/1731363934923/Put/seqid=0 2024-11-11T22:25:35,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741893_1069 (size=12201) 2024-11-11T22:25:35,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T22:25:35,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:25:35,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363995653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:35,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T22:25:35,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:35,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363995664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:35,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/e42c623298224ce3bd50a71636d2ace3 2024-11-11T22:25:35,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e7a840e1a0594912865974b7936146ef as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e7a840e1a0594912865974b7936146ef 2024-11-11T22:25:35,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e7a840e1a0594912865974b7936146ef, entries=200, sequenceid=260, filesize=14.2 K 2024-11-11T22:25:35,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/92725a4904b244b8a7eec64a39080778 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/92725a4904b244b8a7eec64a39080778 2024-11-11T22:25:35,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T22:25:35,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/92725a4904b244b8a7eec64a39080778, entries=150, sequenceid=260, filesize=11.9 K 2024-11-11T22:25:35,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:35,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/e42c623298224ce3bd50a71636d2ace3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e42c623298224ce3bd50a71636d2ace3 2024-11-11T22:25:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:35,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e42c623298224ce3bd50a71636d2ace3, entries=150, sequenceid=260, filesize=11.9 K 2024-11-11T22:25:35,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 765de8aa1edd426d207e09298c569dc6 in 898ms, sequenceid=260, compaction requested=true 2024-11-11T22:25:35,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:35,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:35,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:35,823 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:35,823 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:25:35,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:35,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:35,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:35,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:35,825 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:35,825 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:35,825 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,825 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/f21d8dffd38c4555a1e67aadc7636dac, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/6cea76a3a74a4f2ab996fb95ac294620, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e7a840e1a0594912865974b7936146ef] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=38.5 K 2024-11-11T22:25:35,825 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:35,825 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:35,826 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,826 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/d3f2c215b07e4c9d9e4bec61e8c31e96, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/00356badc91d4b55b1ca74726b2fe7f7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/92725a4904b244b8a7eec64a39080778] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=36.2 K 2024-11-11T22:25:35,826 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting d3f2c215b07e4c9d9e4bec61e8c31e96, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731363933509 2024-11-11T22:25:35,826 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting f21d8dffd38c4555a1e67aadc7636dac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731363933509 2024-11-11T22:25:35,827 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 00356badc91d4b55b1ca74726b2fe7f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731363933612 2024-11-11T22:25:35,827 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cea76a3a74a4f2ab996fb95ac294620, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731363933612 2024-11-11T22:25:35,827 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 92725a4904b244b8a7eec64a39080778, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363934273 2024-11-11T22:25:35,827 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7a840e1a0594912865974b7936146ef, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363934273 2024-11-11T22:25:35,842 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:35,842 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:35,842 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/b2fde91b611e414dac51b6bd510e9d7c is 50, key is test_row_0/B:col10/1731363934923/Put/seqid=0 2024-11-11T22:25:35,843 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e9979a5de8594e1ab99e3df7849a39c2 is 50, key is test_row_0/A:col10/1731363934923/Put/seqid=0 2024-11-11T22:25:35,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741895_1071 (size=12849) 2024-11-11T22:25:35,865 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e9979a5de8594e1ab99e3df7849a39c2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e9979a5de8594e1ab99e3df7849a39c2 2024-11-11T22:25:35,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741894_1070 (size=12849) 2024-11-11T22:25:35,875 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into e9979a5de8594e1ab99e3df7849a39c2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:35,875 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:35,875 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=13, startTime=1731363935823; duration=0sec 2024-11-11T22:25:35,875 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:35,875 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:35,875 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:35,879 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:35,879 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:35,879 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,879 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/b2fde91b611e414dac51b6bd510e9d7c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/b2fde91b611e414dac51b6bd510e9d7c 2024-11-11T22:25:35,879 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/af7b526f44cf4757948f93b1fa43b14b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9a7cd18c13974585acacbd1659df86d7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e42c623298224ce3bd50a71636d2ace3] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=36.2 K 2024-11-11T22:25:35,880 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting af7b526f44cf4757948f93b1fa43b14b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731363933509 2024-11-11T22:25:35,880 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a7cd18c13974585acacbd1659df86d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731363933612 2024-11-11T22:25:35,881 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting e42c623298224ce3bd50a71636d2ace3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363934273 2024-11-11T22:25:35,892 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into b2fde91b611e414dac51b6bd510e9d7c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:35,892 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:35,892 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=13, startTime=1731363935823; duration=0sec 2024-11-11T22:25:35,892 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:35,892 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:35,894 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:35,895 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/dc56951a950142079a6b2791bda304c7 is 50, key is test_row_0/C:col10/1731363934923/Put/seqid=0 2024-11-11T22:25:35,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741896_1072 (size=12849) 2024-11-11T22:25:35,924 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/dc56951a950142079a6b2791bda304c7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/dc56951a950142079a6b2791bda304c7 2024-11-11T22:25:35,933 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into dc56951a950142079a6b2791bda304c7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:35,933 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:35,933 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=13, startTime=1731363935824; duration=0sec 2024-11-11T22:25:35,933 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:35,933 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:35,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T22:25:35,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:35,967 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:25:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/a393e12f75f346feb609e9cd9a4268eb is 50, key is test_row_0/A:col10/1731363935028/Put/seqid=0 2024-11-11T22:25:35,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741897_1073 (size=12301) 2024-11-11T22:25:35,983 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/a393e12f75f346feb609e9cd9a4268eb 2024-11-11T22:25:36,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/cf7617cf6f6e43aba4388595c00111d0 is 50, key is test_row_0/B:col10/1731363935028/Put/seqid=0 2024-11-11T22:25:36,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741898_1074 (size=12301) 2024-11-11T22:25:36,016 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/cf7617cf6f6e43aba4388595c00111d0 2024-11-11T22:25:36,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/7c313a44bf7c48019f2d4341f3e823a7 is 50, key is test_row_0/C:col10/1731363935028/Put/seqid=0 2024-11-11T22:25:36,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741899_1075 (size=12301) 2024-11-11T22:25:36,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:36,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363996182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363996187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363996293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363996304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:25:36,434 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/7c313a44bf7c48019f2d4341f3e823a7 2024-11-11T22:25:36,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/a393e12f75f346feb609e9cd9a4268eb as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a393e12f75f346feb609e9cd9a4268eb 2024-11-11T22:25:36,448 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a393e12f75f346feb609e9cd9a4268eb, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T22:25:36,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/cf7617cf6f6e43aba4388595c00111d0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cf7617cf6f6e43aba4388595c00111d0 2024-11-11T22:25:36,455 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cf7617cf6f6e43aba4388595c00111d0, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T22:25:36,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/7c313a44bf7c48019f2d4341f3e823a7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/7c313a44bf7c48019f2d4341f3e823a7 2024-11-11T22:25:36,462 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/7c313a44bf7c48019f2d4341f3e823a7, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T22:25:36,464 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 765de8aa1edd426d207e09298c569dc6 in 496ms, sequenceid=287, compaction requested=false 2024-11-11T22:25:36,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:36,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:36,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T22:25:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T22:25:36,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T22:25:36,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2690 sec 2024-11-11T22:25:36,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.2760 sec 2024-11-11T22:25:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:36,506 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:36,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T22:25:36,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/a4fd4ec444be4a4d9bf9bfaaaa064b5b is 50, key is test_row_0/A:col10/1731363936180/Put/seqid=0 2024-11-11T22:25:36,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741900_1076 (size=12301) 2024-11-11T22:25:36,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363996562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363996565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363996674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363996674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731363996693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731363996693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,695 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731363996693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,695 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,697 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:36,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363996883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363996883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:36,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/a4fd4ec444be4a4d9bf9bfaaaa064b5b 2024-11-11T22:25:36,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/266481dab8024d38aa850f42440a977c is 50, key is test_row_0/B:col10/1731363936180/Put/seqid=0 2024-11-11T22:25:36,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741901_1077 (size=12301) 2024-11-11T22:25:37,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363997193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:37,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363997193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:37,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/266481dab8024d38aa850f42440a977c 2024-11-11T22:25:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:25:37,343 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:37,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T22:25:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:25:37,346 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:37,347 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:37,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:37,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/d6d0c35a110a41a49cf15360bdb09f31 is 50, key is test_row_0/C:col10/1731363936180/Put/seqid=0 2024-11-11T22:25:37,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741902_1078 (size=12301) 2024-11-11T22:25:37,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/d6d0c35a110a41a49cf15360bdb09f31 2024-11-11T22:25:37,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/a4fd4ec444be4a4d9bf9bfaaaa064b5b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a4fd4ec444be4a4d9bf9bfaaaa064b5b 2024-11-11T22:25:37,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a4fd4ec444be4a4d9bf9bfaaaa064b5b, entries=150, sequenceid=300, filesize=12.0 K 2024-11-11T22:25:37,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/266481dab8024d38aa850f42440a977c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/266481dab8024d38aa850f42440a977c 2024-11-11T22:25:37,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/266481dab8024d38aa850f42440a977c, entries=150, sequenceid=300, filesize=12.0 K 2024-11-11T22:25:37,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/d6d0c35a110a41a49cf15360bdb09f31 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d6d0c35a110a41a49cf15360bdb09f31 2024-11-11T22:25:37,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d6d0c35a110a41a49cf15360bdb09f31, entries=150, sequenceid=300, filesize=12.0 K 2024-11-11T22:25:37,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 765de8aa1edd426d207e09298c569dc6 in 884ms, sequenceid=300, compaction requested=true 2024-11-11T22:25:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:37,390 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:37,390 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:37,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:37,392 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:37,392 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:37,392 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:37,392 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:37,392 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:37,392 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:37,392 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e9979a5de8594e1ab99e3df7849a39c2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a393e12f75f346feb609e9cd9a4268eb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a4fd4ec444be4a4d9bf9bfaaaa064b5b] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=36.6 K 2024-11-11T22:25:37,392 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/b2fde91b611e414dac51b6bd510e9d7c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cf7617cf6f6e43aba4388595c00111d0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/266481dab8024d38aa850f42440a977c] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=36.6 K 2024-11-11T22:25:37,393 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9979a5de8594e1ab99e3df7849a39c2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363934273 2024-11-11T22:25:37,393 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting b2fde91b611e414dac51b6bd510e9d7c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363934273 2024-11-11T22:25:37,393 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting a393e12f75f346feb609e9cd9a4268eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731363935023 2024-11-11T22:25:37,393 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting cf7617cf6f6e43aba4388595c00111d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731363935023 2024-11-11T22:25:37,394 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4fd4ec444be4a4d9bf9bfaaaa064b5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731363936180 2024-11-11T22:25:37,394 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 266481dab8024d38aa850f42440a977c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731363936180 2024-11-11T22:25:37,406 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:37,407 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/ee3ceae0087c4c128efa94526114c05a is 50, key is test_row_0/B:col10/1731363936180/Put/seqid=0 2024-11-11T22:25:37,422 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:37,423 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/5497dc9da226405cb84a8644b08f91b4 is 50, key is test_row_0/A:col10/1731363936180/Put/seqid=0 2024-11-11T22:25:37,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741903_1079 (size=13051) 2024-11-11T22:25:37,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741904_1080 (size=13051) 2024-11-11T22:25:37,451 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/5497dc9da226405cb84a8644b08f91b4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/5497dc9da226405cb84a8644b08f91b4 2024-11-11T22:25:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:25:37,461 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into 5497dc9da226405cb84a8644b08f91b4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:37,461 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:37,461 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=13, startTime=1731363937390; duration=0sec 2024-11-11T22:25:37,461 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:37,461 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:37,461 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:37,464 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:37,464 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:37,464 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:37,464 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/dc56951a950142079a6b2791bda304c7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/7c313a44bf7c48019f2d4341f3e823a7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d6d0c35a110a41a49cf15360bdb09f31] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=36.6 K 2024-11-11T22:25:37,465 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc56951a950142079a6b2791bda304c7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363934273 2024-11-11T22:25:37,465 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c313a44bf7c48019f2d4341f3e823a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731363935023 2024-11-11T22:25:37,466 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6d0c35a110a41a49cf15360bdb09f31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731363936180 2024-11-11T22:25:37,480 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:37,481 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/0b0aef0687664e44b930c2418560395d is 50, key is test_row_0/C:col10/1731363936180/Put/seqid=0 2024-11-11T22:25:37,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741905_1081 (size=13051) 2024-11-11T22:25:37,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T22:25:37,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:37,500 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:25:37,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/4871125d5c3f45d0911c3c40e4e705bc is 50, key is test_row_0/A:col10/1731363936557/Put/seqid=0 2024-11-11T22:25:37,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741906_1082 (size=12301) 2024-11-11T22:25:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:25:37,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:37,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363997725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:37,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363997726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:37,833 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/ee3ceae0087c4c128efa94526114c05a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ee3ceae0087c4c128efa94526114c05a 2024-11-11T22:25:37,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363997833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:37,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363997833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:37,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:37,840 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into ee3ceae0087c4c128efa94526114c05a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:37,840 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:37,840 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=13, startTime=1731363937390; duration=0sec 2024-11-11T22:25:37,840 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:37,840 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:37,904 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/0b0aef0687664e44b930c2418560395d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/0b0aef0687664e44b930c2418560395d 2024-11-11T22:25:37,910 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into 0b0aef0687664e44b930c2418560395d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:37,911 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:37,911 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=13, startTime=1731363937390; duration=0sec 2024-11-11T22:25:37,911 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:37,911 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:37,918 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/4871125d5c3f45d0911c3c40e4e705bc 2024-11-11T22:25:37,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/80f96be673f841c49fb463a98925b1fb is 50, key is test_row_0/B:col10/1731363936557/Put/seqid=0 2024-11-11T22:25:37,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741907_1083 (size=12301) 2024-11-11T22:25:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:25:38,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363998043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363998043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:38,343 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/80f96be673f841c49fb463a98925b1fb 2024-11-11T22:25:38,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/1bae7064f566400f964f53909bab9252 is 50, key is test_row_0/C:col10/1731363936557/Put/seqid=0 2024-11-11T22:25:38,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741908_1084 (size=12301) 2024-11-11T22:25:38,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363998355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:38,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363998355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:38,360 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/1bae7064f566400f964f53909bab9252 2024-11-11T22:25:38,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/4871125d5c3f45d0911c3c40e4e705bc as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/4871125d5c3f45d0911c3c40e4e705bc 2024-11-11T22:25:38,372 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/4871125d5c3f45d0911c3c40e4e705bc, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T22:25:38,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/80f96be673f841c49fb463a98925b1fb as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80f96be673f841c49fb463a98925b1fb 2024-11-11T22:25:38,380 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80f96be673f841c49fb463a98925b1fb, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T22:25:38,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/1bae7064f566400f964f53909bab9252 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1bae7064f566400f964f53909bab9252 2024-11-11T22:25:38,388 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1bae7064f566400f964f53909bab9252, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T22:25:38,389 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 765de8aa1edd426d207e09298c569dc6 in 889ms, sequenceid=324, compaction requested=false 2024-11-11T22:25:38,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:38,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:38,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T22:25:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T22:25:38,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T22:25:38,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0440 sec 2024-11-11T22:25:38,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0520 sec 2024-11-11T22:25:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:25:38,484 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:38,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T22:25:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:25:38,488 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:38,489 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:38,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:25:38,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T22:25:38,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:38,644 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T22:25:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/d819d8a70c904d0f93d56386f4556592 is 50, key is test_row_0/A:col10/1731363937723/Put/seqid=0 2024-11-11T22:25:38,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741909_1085 (size=12301) 2024-11-11T22:25:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:25:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:38,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363998899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:38,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363998901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363999013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363999013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,075 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/d819d8a70c904d0f93d56386f4556592 2024-11-11T22:25:39,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/e8bfa631053d45e5b42a925d7e8b9d5e is 50, key is test_row_0/B:col10/1731363937723/Put/seqid=0 2024-11-11T22:25:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:25:39,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741910_1086 (size=12301) 2024-11-11T22:25:39,136 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/e8bfa631053d45e5b42a925d7e8b9d5e 2024-11-11T22:25:39,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9051051ac6504bbb87357aaf59fc3b3b is 50, key is test_row_0/C:col10/1731363937723/Put/seqid=0 2024-11-11T22:25:39,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741911_1087 (size=12301) 2024-11-11T22:25:39,209 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9051051ac6504bbb87357aaf59fc3b3b 2024-11-11T22:25:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/d819d8a70c904d0f93d56386f4556592 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d819d8a70c904d0f93d56386f4556592 2024-11-11T22:25:39,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363999223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:39,226 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d819d8a70c904d0f93d56386f4556592, entries=150, sequenceid=339, filesize=12.0 K 2024-11-11T22:25:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363999224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/e8bfa631053d45e5b42a925d7e8b9d5e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e8bfa631053d45e5b42a925d7e8b9d5e 2024-11-11T22:25:39,235 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e8bfa631053d45e5b42a925d7e8b9d5e, entries=150, sequenceid=339, filesize=12.0 K 2024-11-11T22:25:39,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9051051ac6504bbb87357aaf59fc3b3b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9051051ac6504bbb87357aaf59fc3b3b 2024-11-11T22:25:39,246 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9051051ac6504bbb87357aaf59fc3b3b, entries=150, sequenceid=339, filesize=12.0 K 2024-11-11T22:25:39,247 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 765de8aa1edd426d207e09298c569dc6 in 603ms, sequenceid=339, compaction requested=true 2024-11-11T22:25:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T22:25:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T22:25:39,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T22:25:39,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 760 msec 2024-11-11T22:25:39,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 767 msec 2024-11-11T22:25:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:39,540 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:39,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:25:39,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/21a88942f2f04081950e865840165405 is 50, key is test_row_0/A:col10/1731363939534/Put/seqid=0 2024-11-11T22:25:39,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741912_1088 (size=12301) 2024-11-11T22:25:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363999559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363999560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:25:39,623 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:39,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T22:25:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:25:39,627 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:39,627 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:39,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:39,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363999673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363999673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:25:39,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T22:25:39,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:39,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:39,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:39,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:39,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:39,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731363999883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731363999883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:39,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T22:25:39,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:39,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:39,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:39,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:39,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:25:39,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/21a88942f2f04081950e865840165405 2024-11-11T22:25:39,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/2f1de4ddbaef4fa8a7bd5611fab72b9c is 50, key is test_row_0/B:col10/1731363939534/Put/seqid=0 2024-11-11T22:25:39,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741913_1089 (size=12301) 2024-11-11T22:25:39,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/2f1de4ddbaef4fa8a7bd5611fab72b9c 2024-11-11T22:25:40,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/5dcd0ab4ea324c1b8c86c777b4f71c65 is 50, key is test_row_0/C:col10/1731363939534/Put/seqid=0 2024-11-11T22:25:40,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741914_1090 (size=12301) 2024-11-11T22:25:40,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/5dcd0ab4ea324c1b8c86c777b4f71c65 2024-11-11T22:25:40,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/21a88942f2f04081950e865840165405 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/21a88942f2f04081950e865840165405 2024-11-11T22:25:40,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/21a88942f2f04081950e865840165405, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T22:25:40,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/2f1de4ddbaef4fa8a7bd5611fab72b9c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2f1de4ddbaef4fa8a7bd5611fab72b9c 2024-11-11T22:25:40,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2f1de4ddbaef4fa8a7bd5611fab72b9c, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T22:25:40,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/5dcd0ab4ea324c1b8c86c777b4f71c65 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/5dcd0ab4ea324c1b8c86c777b4f71c65 2024-11-11T22:25:40,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/5dcd0ab4ea324c1b8c86c777b4f71c65, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T22:25:40,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 765de8aa1edd426d207e09298c569dc6 in 519ms, sequenceid=364, compaction requested=true 2024-11-11T22:25:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:40,060 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:25:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:25:40,061 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:25:40,062 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:25:40,062 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:40,062 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:40,062 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/5497dc9da226405cb84a8644b08f91b4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/4871125d5c3f45d0911c3c40e4e705bc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d819d8a70c904d0f93d56386f4556592, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/21a88942f2f04081950e865840165405] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=48.8 K 2024-11-11T22:25:40,063 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:25:40,063 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:40,063 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:40,063 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ee3ceae0087c4c128efa94526114c05a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80f96be673f841c49fb463a98925b1fb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e8bfa631053d45e5b42a925d7e8b9d5e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2f1de4ddbaef4fa8a7bd5611fab72b9c] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=48.8 K 2024-11-11T22:25:40,063 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 5497dc9da226405cb84a8644b08f91b4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731363936180 2024-11-11T22:25:40,064 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee3ceae0087c4c128efa94526114c05a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731363936180 2024-11-11T22:25:40,064 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4871125d5c3f45d0911c3c40e4e705bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731363936557 2024-11-11T22:25:40,066 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting d819d8a70c904d0f93d56386f4556592, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731363937710 2024-11-11T22:25:40,066 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f96be673f841c49fb463a98925b1fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731363936557 2024-11-11T22:25:40,066 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a88942f2f04081950e865840165405, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731363938898 2024-11-11T22:25:40,067 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8bfa631053d45e5b42a925d7e8b9d5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731363937710 2024-11-11T22:25:40,068 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f1de4ddbaef4fa8a7bd5611fab72b9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731363938898 2024-11-11T22:25:40,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T22:25:40,087 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:40,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:40,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:25:40,087 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/b029456c8b5e4a61ac8954b5813d4d55 is 50, key is test_row_0/A:col10/1731363939534/Put/seqid=0 2024-11-11T22:25:40,095 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:40,095 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/23ffcfdc85624027b1d123ba46fb65e2 is 50, key is test_row_0/B:col10/1731363939534/Put/seqid=0 2024-11-11T22:25:40,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/829496052ed24e6591caeb418f8ab835 is 50, key is test_row_0/A:col10/1731363939553/Put/seqid=0 2024-11-11T22:25:40,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741917_1093 (size=12301) 2024-11-11T22:25:40,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741916_1092 (size=13187) 2024-11-11T22:25:40,125 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/23ffcfdc85624027b1d123ba46fb65e2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/23ffcfdc85624027b1d123ba46fb65e2 2024-11-11T22:25:40,140 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into 23ffcfdc85624027b1d123ba46fb65e2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:40,140 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:40,141 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=12, startTime=1731363940060; duration=0sec 2024-11-11T22:25:40,142 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:40,142 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:40,142 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:25:40,144 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:25:40,144 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:40,144 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:40,144 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/0b0aef0687664e44b930c2418560395d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1bae7064f566400f964f53909bab9252, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9051051ac6504bbb87357aaf59fc3b3b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/5dcd0ab4ea324c1b8c86c777b4f71c65] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=48.8 K 2024-11-11T22:25:40,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741915_1091 (size=13187) 2024-11-11T22:25:40,146 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b0aef0687664e44b930c2418560395d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731363936180 2024-11-11T22:25:40,146 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/829496052ed24e6591caeb418f8ab835 2024-11-11T22:25:40,146 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bae7064f566400f964f53909bab9252, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731363936557 2024-11-11T22:25:40,148 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9051051ac6504bbb87357aaf59fc3b3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731363937710 2024-11-11T22:25:40,149 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dcd0ab4ea324c1b8c86c777b4f71c65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731363938898 2024-11-11T22:25:40,156 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/b029456c8b5e4a61ac8954b5813d4d55 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b029456c8b5e4a61ac8954b5813d4d55 2024-11-11T22:25:40,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/8cf312868f3c4329949c41f1f1954b2e is 50, key is test_row_0/B:col10/1731363939553/Put/seqid=0 2024-11-11T22:25:40,170 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into b029456c8b5e4a61ac8954b5813d4d55(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:40,170 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:40,170 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=12, startTime=1731363940060; duration=0sec 2024-11-11T22:25:40,172 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:40,172 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:40,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741918_1094 (size=12301) 2024-11-11T22:25:40,178 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:40,178 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/8cf312868f3c4329949c41f1f1954b2e 2024-11-11T22:25:40,178 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/71f1a4d90b69438e836f1f3746bb2fbe is 50, key is test_row_0/C:col10/1731363939534/Put/seqid=0 2024-11-11T22:25:40,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/62d66282b57142ea966bc18d8a2c77b5 is 50, key is test_row_0/C:col10/1731363939553/Put/seqid=0 2024-11-11T22:25:40,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741919_1095 (size=13187) 2024-11-11T22:25:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:40,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:40,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741920_1096 (size=12301) 2024-11-11T22:25:40,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364000239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364000242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:25:40,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364000353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364000353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364000563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364000564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,599 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/62d66282b57142ea966bc18d8a2c77b5 2024-11-11T22:25:40,600 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/71f1a4d90b69438e836f1f3746bb2fbe as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71f1a4d90b69438e836f1f3746bb2fbe 2024-11-11T22:25:40,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/829496052ed24e6591caeb418f8ab835 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/829496052ed24e6591caeb418f8ab835 2024-11-11T22:25:40,609 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into 71f1a4d90b69438e836f1f3746bb2fbe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:40,609 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:40,609 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=12, startTime=1731363940060; duration=0sec 2024-11-11T22:25:40,610 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:40,610 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:40,618 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/829496052ed24e6591caeb418f8ab835, entries=150, sequenceid=375, filesize=12.0 K 2024-11-11T22:25:40,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/8cf312868f3c4329949c41f1f1954b2e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/8cf312868f3c4329949c41f1f1954b2e 2024-11-11T22:25:40,626 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/8cf312868f3c4329949c41f1f1954b2e, entries=150, sequenceid=375, filesize=12.0 K 2024-11-11T22:25:40,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/62d66282b57142ea966bc18d8a2c77b5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/62d66282b57142ea966bc18d8a2c77b5 2024-11-11T22:25:40,638 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/62d66282b57142ea966bc18d8a2c77b5, entries=150, sequenceid=375, filesize=12.0 K 2024-11-11T22:25:40,639 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 765de8aa1edd426d207e09298c569dc6 in 552ms, sequenceid=375, compaction requested=false 2024-11-11T22:25:40,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:40,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:40,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T22:25:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T22:25:40,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T22:25:40,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0130 sec 2024-11-11T22:25:40,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.0200 sec 2024-11-11T22:25:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:40,727 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:40,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T22:25:40,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/8ddc5d41d84a4004b5d02e6b48426658 is 50, key is test_row_0/A:col10/1731363940240/Put/seqid=0 2024-11-11T22:25:40,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364000744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37740 deadline: 1731364000743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,748 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741921_1097 (size=12301) 2024-11-11T22:25:40,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364000750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:25:40,763 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:40,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-11T22:25:40,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:25:40,765 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:40,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:40,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364000853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364000863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:25:40,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364000873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364000873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:40,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T22:25:40,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:40,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:40,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:40,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:40,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364001063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:41,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T22:25:41,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:41,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364001073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:25:41,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/8ddc5d41d84a4004b5d02e6b48426658 2024-11-11T22:25:41,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/80e02b35a5384dd18f4de6aba26d8fa4 is 50, key is test_row_0/B:col10/1731363940240/Put/seqid=0 2024-11-11T22:25:41,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741922_1098 (size=12301) 2024-11-11T22:25:41,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/80e02b35a5384dd18f4de6aba26d8fa4 2024-11-11T22:25:41,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/71a15af38db142cda372836459723c46 is 50, key is test_row_0/C:col10/1731363940240/Put/seqid=0 2024-11-11T22:25:41,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741923_1099 (size=12301) 2024-11-11T22:25:41,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T22:25:41,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:41,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T22:25:41,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:41,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364001381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:41,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:41,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364001383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364001383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364001387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:25:41,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T22:25:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:41,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:41,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/71a15af38db142cda372836459723c46 2024-11-11T22:25:41,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/8ddc5d41d84a4004b5d02e6b48426658 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8ddc5d41d84a4004b5d02e6b48426658 2024-11-11T22:25:41,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8ddc5d41d84a4004b5d02e6b48426658, entries=150, sequenceid=404, filesize=12.0 K 2024-11-11T22:25:41,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/80e02b35a5384dd18f4de6aba26d8fa4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80e02b35a5384dd18f4de6aba26d8fa4 2024-11-11T22:25:41,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80e02b35a5384dd18f4de6aba26d8fa4, entries=150, sequenceid=404, filesize=12.0 K 2024-11-11T22:25:41,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/71a15af38db142cda372836459723c46 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71a15af38db142cda372836459723c46 2024-11-11T22:25:41,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71a15af38db142cda372836459723c46, entries=150, sequenceid=404, filesize=12.0 K 2024-11-11T22:25:41,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 765de8aa1edd426d207e09298c569dc6 in 903ms, sequenceid=404, compaction requested=true 2024-11-11T22:25:41,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:41,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:41,630 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:41,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:41,630 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:41,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:41,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:41,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:41,632 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:41,632 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:41,632 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,632 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b029456c8b5e4a61ac8954b5813d4d55, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/829496052ed24e6591caeb418f8ab835, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8ddc5d41d84a4004b5d02e6b48426658] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=36.9 K 2024-11-11T22:25:41,632 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:41,633 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:41,633 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,633 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/23ffcfdc85624027b1d123ba46fb65e2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/8cf312868f3c4329949c41f1f1954b2e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80e02b35a5384dd18f4de6aba26d8fa4] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=36.9 K 2024-11-11T22:25:41,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:41,633 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b029456c8b5e4a61ac8954b5813d4d55, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731363938898 2024-11-11T22:25:41,633 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 23ffcfdc85624027b1d123ba46fb65e2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731363938898 2024-11-11T22:25:41,635 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cf312868f3c4329949c41f1f1954b2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731363939553 2024-11-11T22:25:41,635 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 829496052ed24e6591caeb418f8ab835, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731363939553 2024-11-11T22:25:41,636 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 80e02b35a5384dd18f4de6aba26d8fa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731363940232 2024-11-11T22:25:41,636 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ddc5d41d84a4004b5d02e6b48426658, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731363940232 2024-11-11T22:25:41,648 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:41,649 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/add2f8d1125a401289f52f146722b1e9 is 50, key is test_row_0/B:col10/1731363940240/Put/seqid=0 2024-11-11T22:25:41,654 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:41,655 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/b3f0bc684e834abc863a119ee1e4c7de is 50, key is test_row_0/A:col10/1731363940240/Put/seqid=0 2024-11-11T22:25:41,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741925_1101 (size=13289) 2024-11-11T22:25:41,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741924_1100 (size=13289) 2024-11-11T22:25:41,673 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/b3f0bc684e834abc863a119ee1e4c7de as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b3f0bc684e834abc863a119ee1e4c7de 2024-11-11T22:25:41,680 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into b3f0bc684e834abc863a119ee1e4c7de(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:41,681 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:41,681 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=13, startTime=1731363941630; duration=0sec 2024-11-11T22:25:41,681 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:41,681 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:41,681 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:41,682 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:41,683 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:41,683 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,683 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71f1a4d90b69438e836f1f3746bb2fbe, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/62d66282b57142ea966bc18d8a2c77b5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71a15af38db142cda372836459723c46] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=36.9 K 2024-11-11T22:25:41,684 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f1a4d90b69438e836f1f3746bb2fbe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731363938898 2024-11-11T22:25:41,685 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62d66282b57142ea966bc18d8a2c77b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731363939553 2024-11-11T22:25:41,685 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71a15af38db142cda372836459723c46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731363940232 2024-11-11T22:25:41,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T22:25:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:41,693 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-11T22:25:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/d65ac6f93b7a41fbbf950ca46b40045b is 50, key is test_row_0/A:col10/1731363940740/Put/seqid=0 2024-11-11T22:25:41,704 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#90 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:41,704 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/4a307c724b6c4f809ef3520a63d5def1 is 50, key is test_row_0/C:col10/1731363940240/Put/seqid=0 2024-11-11T22:25:41,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741926_1102 (size=12301) 2024-11-11T22:25:41,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741927_1103 (size=13289) 2024-11-11T22:25:41,750 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/4a307c724b6c4f809ef3520a63d5def1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4a307c724b6c4f809ef3520a63d5def1 2024-11-11T22:25:41,760 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into 4a307c724b6c4f809ef3520a63d5def1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:41,760 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:41,760 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=13, startTime=1731363941631; duration=0sec 2024-11-11T22:25:41,760 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:41,760 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:41,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:25:41,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364001942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:41,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364001945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364002054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364002055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,076 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/add2f8d1125a401289f52f146722b1e9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/add2f8d1125a401289f52f146722b1e9 2024-11-11T22:25:42,083 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into add2f8d1125a401289f52f146722b1e9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:42,083 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:42,083 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=13, startTime=1731363941630; duration=0sec 2024-11-11T22:25:42,083 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:42,083 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:42,125 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/d65ac6f93b7a41fbbf950ca46b40045b 2024-11-11T22:25:42,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/2ec9f8c1e30b4009921d02a8a458d98a is 50, key is test_row_0/B:col10/1731363940740/Put/seqid=0 2024-11-11T22:25:42,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741928_1104 (size=12301) 2024-11-11T22:25:42,140 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/2ec9f8c1e30b4009921d02a8a458d98a 2024-11-11T22:25:42,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/6d0d747dacff402fadf5cc78efcc3630 is 50, key is test_row_0/C:col10/1731363940740/Put/seqid=0 2024-11-11T22:25:42,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741929_1105 (size=12301) 2024-11-11T22:25:42,162 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/6d0d747dacff402fadf5cc78efcc3630 2024-11-11T22:25:42,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/d65ac6f93b7a41fbbf950ca46b40045b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d65ac6f93b7a41fbbf950ca46b40045b 2024-11-11T22:25:42,180 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d65ac6f93b7a41fbbf950ca46b40045b, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T22:25:42,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/2ec9f8c1e30b4009921d02a8a458d98a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2ec9f8c1e30b4009921d02a8a458d98a 2024-11-11T22:25:42,195 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2ec9f8c1e30b4009921d02a8a458d98a, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T22:25:42,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/6d0d747dacff402fadf5cc78efcc3630 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/6d0d747dacff402fadf5cc78efcc3630 2024-11-11T22:25:42,208 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/6d0d747dacff402fadf5cc78efcc3630, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T22:25:42,209 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 765de8aa1edd426d207e09298c569dc6 in 517ms, sequenceid=415, compaction requested=false 2024-11-11T22:25:42,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:42,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:42,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-11T22:25:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-11T22:25:42,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T22:25:42,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4450 sec 2024-11-11T22:25:42,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4510 sec 2024-11-11T22:25:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:42,273 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:42,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-11T22:25:42,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364002276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364002278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/b070c3c559bd49168b53552a9ab828ce is 50, key is test_row_0/A:col10/1731363942273/Put/seqid=0 2024-11-11T22:25:42,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741930_1106 (size=14741) 2024-11-11T22:25:42,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/b070c3c559bd49168b53552a9ab828ce 2024-11-11T22:25:42,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/12b1b449e9db493a8a977ad3ba58c032 is 50, key is test_row_0/B:col10/1731363942273/Put/seqid=0 2024-11-11T22:25:42,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741931_1107 (size=12301) 2024-11-11T22:25:42,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/12b1b449e9db493a8a977ad3ba58c032 2024-11-11T22:25:42,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/b3aea0ee98fb45889c744bcb3447cbb6 is 50, key is test_row_0/C:col10/1731363942273/Put/seqid=0 2024-11-11T22:25:42,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741932_1108 (size=12301) 2024-11-11T22:25:42,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364002383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364002383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364002393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364002393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,397 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,398 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364002593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364002593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/b3aea0ee98fb45889c744bcb3447cbb6 2024-11-11T22:25:42,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/b070c3c559bd49168b53552a9ab828ce as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b070c3c559bd49168b53552a9ab828ce 2024-11-11T22:25:42,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b070c3c559bd49168b53552a9ab828ce, entries=200, sequenceid=449, filesize=14.4 K 2024-11-11T22:25:42,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/12b1b449e9db493a8a977ad3ba58c032 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/12b1b449e9db493a8a977ad3ba58c032 2024-11-11T22:25:42,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/12b1b449e9db493a8a977ad3ba58c032, entries=150, sequenceid=449, filesize=12.0 K 2024-11-11T22:25:42,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/b3aea0ee98fb45889c744bcb3447cbb6 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b3aea0ee98fb45889c744bcb3447cbb6 2024-11-11T22:25:42,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b3aea0ee98fb45889c744bcb3447cbb6, entries=150, sequenceid=449, filesize=12.0 K 2024-11-11T22:25:42,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 765de8aa1edd426d207e09298c569dc6 in 508ms, sequenceid=449, compaction requested=true 2024-11-11T22:25:42,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:42,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:42,782 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:42,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:42,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:42,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:42,782 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:42,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:42,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:42,783 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:42,783 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:42,783 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:42,783 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:42,783 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:42,783 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:42,784 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b3f0bc684e834abc863a119ee1e4c7de, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d65ac6f93b7a41fbbf950ca46b40045b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b070c3c559bd49168b53552a9ab828ce] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=39.4 K 2024-11-11T22:25:42,784 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/add2f8d1125a401289f52f146722b1e9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2ec9f8c1e30b4009921d02a8a458d98a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/12b1b449e9db493a8a977ad3ba58c032] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=37.0 K 2024-11-11T22:25:42,784 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting add2f8d1125a401289f52f146722b1e9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731363940232 2024-11-11T22:25:42,784 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3f0bc684e834abc863a119ee1e4c7de, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731363940232 2024-11-11T22:25:42,784 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting d65ac6f93b7a41fbbf950ca46b40045b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731363940731 2024-11-11T22:25:42,784 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec9f8c1e30b4009921d02a8a458d98a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731363940731 2024-11-11T22:25:42,785 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b070c3c559bd49168b53552a9ab828ce, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731363942265 2024-11-11T22:25:42,785 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 12b1b449e9db493a8a977ad3ba58c032, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731363942265 2024-11-11T22:25:42,793 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:42,793 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/721ea01643df47e7a4174200d6246627 is 50, key is test_row_0/A:col10/1731363942273/Put/seqid=0 2024-11-11T22:25:42,794 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:42,794 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/85ad1364099c446ba538e95e12a718d5 is 50, key is test_row_0/B:col10/1731363942273/Put/seqid=0 2024-11-11T22:25:42,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741933_1109 (size=13391) 2024-11-11T22:25:42,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741934_1110 (size=13391) 2024-11-11T22:25:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:42,913 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:42,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:25:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:25:42,913 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:42,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-11T22:25:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:25:42,916 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:42,917 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:42,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:42,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e8b9ea2ec3b04aa3933a00b4362cbec1 is 50, key is test_row_0/A:col10/1731363942912/Put/seqid=0 2024-11-11T22:25:42,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364002967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364002970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:42,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741935_1111 (size=17181) 2024-11-11T22:25:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:25:43,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:25:43,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364003083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:43,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:43,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364003083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:43,209 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/85ad1364099c446ba538e95e12a718d5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/85ad1364099c446ba538e95e12a718d5 2024-11-11T22:25:43,209 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/721ea01643df47e7a4174200d6246627 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/721ea01643df47e7a4174200d6246627 2024-11-11T22:25:43,219 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into 721ea01643df47e7a4174200d6246627(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:43,219 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:43,219 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=13, startTime=1731363942782; duration=0sec 2024-11-11T22:25:43,219 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:43,219 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:43,219 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into 85ad1364099c446ba538e95e12a718d5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:43,219 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:43,219 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:25:43,219 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=13, startTime=1731363942782; duration=0sec 2024-11-11T22:25:43,220 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:43,220 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:43,221 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:25:43,221 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:43,221 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,221 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4a307c724b6c4f809ef3520a63d5def1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/6d0d747dacff402fadf5cc78efcc3630, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b3aea0ee98fb45889c744bcb3447cbb6] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=37.0 K 2024-11-11T22:25:43,221 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a307c724b6c4f809ef3520a63d5def1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731363940232 2024-11-11T22:25:43,223 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d0d747dacff402fadf5cc78efcc3630, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731363940731 2024-11-11T22:25:43,223 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3aea0ee98fb45889c744bcb3447cbb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731363942265 2024-11-11T22:25:43,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:25:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:25:43,238 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:43,239 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/3f4045850e674066a44b1186e6369c1b is 50, key is test_row_0/C:col10/1731363942273/Put/seqid=0 2024-11-11T22:25:43,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741936_1112 (size=13391) 2024-11-11T22:25:43,274 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/3f4045850e674066a44b1186e6369c1b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/3f4045850e674066a44b1186e6369c1b 2024-11-11T22:25:43,283 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into 3f4045850e674066a44b1186e6369c1b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:43,283 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:43,283 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=13, startTime=1731363942782; duration=0sec 2024-11-11T22:25:43,283 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:43,284 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:43,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364003293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364003293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:43,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:25:43,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:43,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e8b9ea2ec3b04aa3933a00b4362cbec1 2024-11-11T22:25:43,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/921671bf56254e26b7fa560e79b66b30 is 50, key is test_row_0/B:col10/1731363942912/Put/seqid=0 2024-11-11T22:25:43,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741937_1113 (size=12301) 2024-11-11T22:25:43,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:25:43,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:25:43,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364003603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:43,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364003603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:43,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:25:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/921671bf56254e26b7fa560e79b66b30 2024-11-11T22:25:43,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/be3e1070198545469785884eb19a40ff is 50, key is test_row_0/C:col10/1731363942912/Put/seqid=0 2024-11-11T22:25:43,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741938_1114 (size=12301) 2024-11-11T22:25:43,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:25:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:43,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:44,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:25:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:44,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:44,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:25:44,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364004113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364004113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:44,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:25:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:44,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:44,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/be3e1070198545469785884eb19a40ff 2024-11-11T22:25:44,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/e8b9ea2ec3b04aa3933a00b4362cbec1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e8b9ea2ec3b04aa3933a00b4362cbec1 2024-11-11T22:25:44,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e8b9ea2ec3b04aa3933a00b4362cbec1, entries=250, sequenceid=461, filesize=16.8 K 2024-11-11T22:25:44,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/921671bf56254e26b7fa560e79b66b30 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/921671bf56254e26b7fa560e79b66b30 2024-11-11T22:25:44,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/921671bf56254e26b7fa560e79b66b30, entries=150, sequenceid=461, filesize=12.0 K 2024-11-11T22:25:44,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/be3e1070198545469785884eb19a40ff as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/be3e1070198545469785884eb19a40ff 2024-11-11T22:25:44,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/be3e1070198545469785884eb19a40ff, entries=150, sequenceid=461, filesize=12.0 K 2024-11-11T22:25:44,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 765de8aa1edd426d207e09298c569dc6 in 1341ms, sequenceid=461, compaction requested=false 2024-11-11T22:25:44,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:44,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:25:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:44,309 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:25:44,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/0581566ca31445f1b51c443458ad7edb is 50, key is test_row_0/A:col10/1731363942967/Put/seqid=0 2024-11-11T22:25:44,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741939_1115 (size=12301) 2024-11-11T22:25:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:44,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:44,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364004419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:44,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364004421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:44,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364004533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364004533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:44,728 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/0581566ca31445f1b51c443458ad7edb 2024-11-11T22:25:44,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/4b1ee3b18dbe40fa8aae0810c05a3bce is 50, key is test_row_0/B:col10/1731363942967/Put/seqid=0 2024-11-11T22:25:44,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741940_1116 (size=12301) 2024-11-11T22:25:44,740 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/4b1ee3b18dbe40fa8aae0810c05a3bce 2024-11-11T22:25:44,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364004743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:44,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364004743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:44,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/1cd6365964a840aaae82a33e689ee311 is 50, key is test_row_0/C:col10/1731363942967/Put/seqid=0 2024-11-11T22:25:44,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741941_1117 (size=12301) 2024-11-11T22:25:44,758 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/1cd6365964a840aaae82a33e689ee311 2024-11-11T22:25:44,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/0581566ca31445f1b51c443458ad7edb as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/0581566ca31445f1b51c443458ad7edb 2024-11-11T22:25:44,769 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/0581566ca31445f1b51c443458ad7edb, entries=150, sequenceid=488, filesize=12.0 K 2024-11-11T22:25:44,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/4b1ee3b18dbe40fa8aae0810c05a3bce as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/4b1ee3b18dbe40fa8aae0810c05a3bce 2024-11-11T22:25:44,774 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/4b1ee3b18dbe40fa8aae0810c05a3bce, entries=150, sequenceid=488, filesize=12.0 K 2024-11-11T22:25:44,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/1cd6365964a840aaae82a33e689ee311 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1cd6365964a840aaae82a33e689ee311 2024-11-11T22:25:44,780 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1cd6365964a840aaae82a33e689ee311, entries=150, sequenceid=488, filesize=12.0 K 2024-11-11T22:25:44,782 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 765de8aa1edd426d207e09298c569dc6 in 474ms, sequenceid=488, compaction requested=true 2024-11-11T22:25:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-11T22:25:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-11T22:25:44,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T22:25:44,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-11T22:25:44,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.8720 sec 2024-11-11T22:25:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:45,057 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:45,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T22:25:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:25:45,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/10e6dfa58dfd46db8483ae56eae2138e is 50, key is test_row_0/A:col10/1731363945056/Put/seqid=0 2024-11-11T22:25:45,063 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:45,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-11T22:25:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:25:45,066 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:45,066 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:45,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:45,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741942_1118 (size=17181) 2024-11-11T22:25:45,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/10e6dfa58dfd46db8483ae56eae2138e 2024-11-11T22:25:45,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/65d054d58d3a475bb932e7abc0eb6fb1 is 50, key is test_row_0/B:col10/1731363945056/Put/seqid=0 2024-11-11T22:25:45,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741943_1119 (size=12301) 2024-11-11T22:25:45,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364005115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:45,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364005117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:45,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37682 deadline: 1731364005123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:45,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37696 deadline: 1731364005123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:45,125 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:45,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,125 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 765de8aa1edd426d207e09298c569dc6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:45,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:25:45,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:25:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364005223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364005223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:45,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:25:45,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:45,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:25:45,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364005433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:45,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364005433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:45,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/65d054d58d3a475bb932e7abc0eb6fb1 2024-11-11T22:25:45,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/f8500a5455c84c83921605aa48e996cc is 50, key is test_row_0/C:col10/1731363945056/Put/seqid=0 2024-11-11T22:25:45,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:25:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741944_1120 (size=12301) 2024-11-11T22:25:45,545 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T22:25:45,546 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:25:45,547 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T22:25:45,547 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:25:45,550 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T22:25:45,551 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:25:45,551 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:45,551 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:45,551 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:45,551 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:45,551 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:45,551 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:45,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:25:45,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:25:45,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:25:45,554 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T22:25:45,554 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:25:45,554 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:45,554 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:45,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:25:45,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:25:45,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:25:45,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:45,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37668 deadline: 1731364005743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37728 deadline: 1731364005744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:25:45,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:25:45,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:45,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:45,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/f8500a5455c84c83921605aa48e996cc 2024-11-11T22:25:45,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/10e6dfa58dfd46db8483ae56eae2138e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/10e6dfa58dfd46db8483ae56eae2138e 2024-11-11T22:25:45,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/10e6dfa58dfd46db8483ae56eae2138e, entries=250, sequenceid=502, filesize=16.8 K 2024-11-11T22:25:45,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/65d054d58d3a475bb932e7abc0eb6fb1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/65d054d58d3a475bb932e7abc0eb6fb1 2024-11-11T22:25:45,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/65d054d58d3a475bb932e7abc0eb6fb1, entries=150, sequenceid=502, filesize=12.0 K 2024-11-11T22:25:45,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/f8500a5455c84c83921605aa48e996cc as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f8500a5455c84c83921605aa48e996cc 2024-11-11T22:25:45,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f8500a5455c84c83921605aa48e996cc, entries=150, sequenceid=502, filesize=12.0 K 2024-11-11T22:25:45,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 765de8aa1edd426d207e09298c569dc6 in 906ms, sequenceid=502, compaction requested=true 2024-11-11T22:25:45,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:45,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:25:45,963 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:25:45,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:45,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:25:45,964 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:25:45,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:45,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 765de8aa1edd426d207e09298c569dc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:25:45,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:45,965 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:25:45,965 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:25:45,965 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/A is initiating minor compaction (all files) 2024-11-11T22:25:45,965 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/B is initiating minor compaction (all files) 2024-11-11T22:25:45,965 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/A in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,965 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/B in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,965 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/85ad1364099c446ba538e95e12a718d5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/921671bf56254e26b7fa560e79b66b30, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/4b1ee3b18dbe40fa8aae0810c05a3bce, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/65d054d58d3a475bb932e7abc0eb6fb1] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=49.1 K 2024-11-11T22:25:45,965 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/721ea01643df47e7a4174200d6246627, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e8b9ea2ec3b04aa3933a00b4362cbec1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/0581566ca31445f1b51c443458ad7edb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/10e6dfa58dfd46db8483ae56eae2138e] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=58.6 K 2024-11-11T22:25:45,966 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721ea01643df47e7a4174200d6246627, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731363942265 2024-11-11T22:25:45,966 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 85ad1364099c446ba538e95e12a718d5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731363942265 2024-11-11T22:25:45,966 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 921671bf56254e26b7fa560e79b66b30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731363942907 2024-11-11T22:25:45,966 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8b9ea2ec3b04aa3933a00b4362cbec1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731363942907 2024-11-11T22:25:45,967 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1ee3b18dbe40fa8aae0810c05a3bce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731363942959 2024-11-11T22:25:45,967 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0581566ca31445f1b51c443458ad7edb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731363942959 2024-11-11T22:25:45,967 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d054d58d3a475bb932e7abc0eb6fb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731363944414 2024-11-11T22:25:45,967 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10e6dfa58dfd46db8483ae56eae2138e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731363944414 2024-11-11T22:25:45,978 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#B#compaction#108 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:45,978 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/ccd576e7c0b344dc8c5bbfc10b3eac8c is 50, key is test_row_0/B:col10/1731363945056/Put/seqid=0 2024-11-11T22:25:45,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741945_1121 (size=13527) 2024-11-11T22:25:45,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:25:45,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:45,990 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:25:46,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/3104cae5af5d48d19adfc60f29bdd15f is 50, key is test_row_0/A:col10/1731363945113/Put/seqid=0 2024-11-11T22:25:46,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741946_1122 (size=12301) 2024-11-11T22:25:46,015 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/3104cae5af5d48d19adfc60f29bdd15f 2024-11-11T22:25:46,021 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#A#compaction#109 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:46,021 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/5000eff09aec47ce9c0c17e26cce461d is 50, key is test_row_0/A:col10/1731363945056/Put/seqid=0 2024-11-11T22:25:46,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/a5baadf168bd459a97d9f49559bed3ea is 50, key is test_row_0/B:col10/1731363945113/Put/seqid=0 2024-11-11T22:25:46,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741947_1123 (size=13527) 2024-11-11T22:25:46,032 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:25:46,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741948_1124 (size=12301) 2024-11-11T22:25:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:25:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:46,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. as already flushing 2024-11-11T22:25:46,255 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T22:25:46,255 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T22:25:46,255 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:25:46,255 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:25:46,255 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:46,255 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:46,255 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:46,255 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:46,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:25:46,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:25:46,395 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/ccd576e7c0b344dc8c5bbfc10b3eac8c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ccd576e7c0b344dc8c5bbfc10b3eac8c 2024-11-11T22:25:46,402 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/B of 765de8aa1edd426d207e09298c569dc6 into ccd576e7c0b344dc8c5bbfc10b3eac8c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:46,402 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:46,402 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/B, priority=12, startTime=1731363945964; duration=0sec 2024-11-11T22:25:46,402 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:25:46,402 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:B 2024-11-11T22:25:46,402 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:25:46,404 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:25:46,404 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 765de8aa1edd426d207e09298c569dc6/C is initiating minor compaction (all files) 2024-11-11T22:25:46,405 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 765de8aa1edd426d207e09298c569dc6/C in TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:46,405 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/3f4045850e674066a44b1186e6369c1b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/be3e1070198545469785884eb19a40ff, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1cd6365964a840aaae82a33e689ee311, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f8500a5455c84c83921605aa48e996cc] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp, totalSize=49.1 K 2024-11-11T22:25:46,405 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f4045850e674066a44b1186e6369c1b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731363942265 2024-11-11T22:25:46,406 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting be3e1070198545469785884eb19a40ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731363942907 2024-11-11T22:25:46,406 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cd6365964a840aaae82a33e689ee311, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731363942959 2024-11-11T22:25:46,407 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f8500a5455c84c83921605aa48e996cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731363944414 2024-11-11T22:25:46,417 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 765de8aa1edd426d207e09298c569dc6#C#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:25:46,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/b124ccf800244b2db3f391947f8c5bac is 50, key is test_row_0/C:col10/1731363945056/Put/seqid=0 2024-11-11T22:25:46,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741949_1125 (size=13527) 2024-11-11T22:25:46,433 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/a5baadf168bd459a97d9f49559bed3ea 2024-11-11T22:25:46,437 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/5000eff09aec47ce9c0c17e26cce461d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/5000eff09aec47ce9c0c17e26cce461d 2024-11-11T22:25:46,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/ca90c023f74b4c4ab2082f4c5a6943a3 is 50, key is test_row_0/C:col10/1731363945113/Put/seqid=0 2024-11-11T22:25:46,442 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/A of 765de8aa1edd426d207e09298c569dc6 into 5000eff09aec47ce9c0c17e26cce461d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:46,442 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:46,442 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/A, priority=12, startTime=1731363945963; duration=0sec 2024-11-11T22:25:46,442 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:46,442 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:A 2024-11-11T22:25:46,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741950_1126 (size=12301) 2024-11-11T22:25:46,834 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/b124ccf800244b2db3f391947f8c5bac as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b124ccf800244b2db3f391947f8c5bac 2024-11-11T22:25:46,842 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 765de8aa1edd426d207e09298c569dc6/C of 765de8aa1edd426d207e09298c569dc6 into b124ccf800244b2db3f391947f8c5bac(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:25:46,842 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:46,842 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., storeName=765de8aa1edd426d207e09298c569dc6/C, priority=12, startTime=1731363945964; duration=0sec 2024-11-11T22:25:46,842 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:25:46,842 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 765de8aa1edd426d207e09298c569dc6:C 2024-11-11T22:25:46,845 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/ca90c023f74b4c4ab2082f4c5a6943a3 2024-11-11T22:25:46,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/3104cae5af5d48d19adfc60f29bdd15f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/3104cae5af5d48d19adfc60f29bdd15f 2024-11-11T22:25:46,855 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/3104cae5af5d48d19adfc60f29bdd15f, entries=150, sequenceid=525, filesize=12.0 K 2024-11-11T22:25:46,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/a5baadf168bd459a97d9f49559bed3ea as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/a5baadf168bd459a97d9f49559bed3ea 2024-11-11T22:25:46,860 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/a5baadf168bd459a97d9f49559bed3ea, entries=150, sequenceid=525, filesize=12.0 K 2024-11-11T22:25:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/ca90c023f74b4c4ab2082f4c5a6943a3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/ca90c023f74b4c4ab2082f4c5a6943a3 2024-11-11T22:25:46,865 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/ca90c023f74b4c4ab2082f4c5a6943a3, entries=150, sequenceid=525, filesize=12.0 K 2024-11-11T22:25:46,866 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=13.42 KB/13740 for 765de8aa1edd426d207e09298c569dc6 in 876ms, sequenceid=525, compaction requested=false 2024-11-11T22:25:46,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 765de8aa1edd426d207e09298c569dc6: 2024-11-11T22:25:46,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:46,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-11T22:25:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-11T22:25:46,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T22:25:46,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-11-11T22:25:46,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.8060 sec 2024-11-11T22:25:47,148 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T22:25:47,148 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T22:25:47,148 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:25:47,148 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:47,148 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:25:47,148 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:47,149 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:47,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:25:47,149 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:47,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:25:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:25:47,225 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:50,824 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T22:25:50,824 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:25:50,824 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:50,825 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:50,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3722 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3610 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1688 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5057 rows 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1685 2024-11-11T22:25:50,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5048 rows 2024-11-11T22:25:50,826 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:25:50,826 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:25:50,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:50,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:50,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:25:50,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:25:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:25:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:25:50,837 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363950837"}]},"ts":"1731363950837"} 2024-11-11T22:25:50,839 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:25:50,840 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:25:50,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:25:50,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=765de8aa1edd426d207e09298c569dc6, UNASSIGN}] 2024-11-11T22:25:50,848 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=765de8aa1edd426d207e09298c569dc6, UNASSIGN 2024-11-11T22:25:50,850 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=765de8aa1edd426d207e09298c569dc6, regionState=CLOSING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:50,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=765de8aa1edd426d207e09298c569dc6, UNASSIGN because future has completed 2024-11-11T22:25:50,854 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:25:50,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:25:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:25:51,011 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:51,011 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:25:51,012 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 765de8aa1edd426d207e09298c569dc6, disabling compactions & flushes 2024-11-11T22:25:51,012 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:51,012 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:51,012 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. after waiting 0 ms 2024-11-11T22:25:51,012 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:51,012 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 765de8aa1edd426d207e09298c569dc6 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T22:25:51,017 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/affd14b5a9b64ebaaef08e07a072175f is 50, key is test_row_0/A:col10/1731363947144/Put/seqid=0 2024-11-11T22:25:51,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741951_1127 (size=12301) 2024-11-11T22:25:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:25:51,427 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/affd14b5a9b64ebaaef08e07a072175f 2024-11-11T22:25:51,437 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/9a9fe03a820e484a8ba24f3fe5fa3378 is 50, key is test_row_0/B:col10/1731363947144/Put/seqid=0 2024-11-11T22:25:51,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741952_1128 (size=12301) 2024-11-11T22:25:51,447 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/9a9fe03a820e484a8ba24f3fe5fa3378 2024-11-11T22:25:51,455 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9a5f55273d8a4152b22834bdb5e35320 is 50, key is test_row_0/C:col10/1731363947144/Put/seqid=0 2024-11-11T22:25:51,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741953_1129 (size=12301) 2024-11-11T22:25:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:25:51,864 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9a5f55273d8a4152b22834bdb5e35320 2024-11-11T22:25:51,869 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/A/affd14b5a9b64ebaaef08e07a072175f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/affd14b5a9b64ebaaef08e07a072175f 2024-11-11T22:25:51,875 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/affd14b5a9b64ebaaef08e07a072175f, entries=150, sequenceid=536, filesize=12.0 K 2024-11-11T22:25:51,876 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/B/9a9fe03a820e484a8ba24f3fe5fa3378 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/9a9fe03a820e484a8ba24f3fe5fa3378 2024-11-11T22:25:51,882 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/9a9fe03a820e484a8ba24f3fe5fa3378, entries=150, sequenceid=536, filesize=12.0 K 2024-11-11T22:25:51,883 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/.tmp/C/9a5f55273d8a4152b22834bdb5e35320 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9a5f55273d8a4152b22834bdb5e35320 2024-11-11T22:25:51,887 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9a5f55273d8a4152b22834bdb5e35320, entries=150, sequenceid=536, filesize=12.0 K 2024-11-11T22:25:51,888 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 765de8aa1edd426d207e09298c569dc6 in 876ms, sequenceid=536, compaction requested=true 2024-11-11T22:25:51,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/30d2d97281ef4e70a22de3e2b3674109, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/29d5f51210ba4d6f8dfac29a0327dd44, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c5af2a140d024c8593716d5f71a83fbf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/ed80ea7c5d4043de9414bfb99d6fe219, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/cba64166e777475a9176acc9ad483969, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/05bcdf107cbf4c9d91932e50a884923e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8e8c9e661906423980d1120b1105acaf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e589c5556a1349559380cee995023238, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c65dfb7fa36d41d9a966bd173b61473e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/7702e3d74c1b43b686aff98c156a806e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/eac2c030dcce4c728db3ba2ea724a5c8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/41beb201d8f4408eaa00be0ca53fdb9e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/97d9c46de7644111959a58748f7e8f19, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/1395214b2ac347d8a325b1ac56c06007, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/185aec4118c847d8825fc5338d72af8f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/79e5e9c94e824d148d394324041ba70d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/f21d8dffd38c4555a1e67aadc7636dac, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/6cea76a3a74a4f2ab996fb95ac294620, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e7a840e1a0594912865974b7936146ef, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e9979a5de8594e1ab99e3df7849a39c2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a393e12f75f346feb609e9cd9a4268eb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/5497dc9da226405cb84a8644b08f91b4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a4fd4ec444be4a4d9bf9bfaaaa064b5b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/4871125d5c3f45d0911c3c40e4e705bc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d819d8a70c904d0f93d56386f4556592, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b029456c8b5e4a61ac8954b5813d4d55, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/21a88942f2f04081950e865840165405, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/829496052ed24e6591caeb418f8ab835, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b3f0bc684e834abc863a119ee1e4c7de, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8ddc5d41d84a4004b5d02e6b48426658, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d65ac6f93b7a41fbbf950ca46b40045b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b070c3c559bd49168b53552a9ab828ce, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/721ea01643df47e7a4174200d6246627, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e8b9ea2ec3b04aa3933a00b4362cbec1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/0581566ca31445f1b51c443458ad7edb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/10e6dfa58dfd46db8483ae56eae2138e] to archive 2024-11-11T22:25:51,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:25:51,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/30d2d97281ef4e70a22de3e2b3674109 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/30d2d97281ef4e70a22de3e2b3674109 2024-11-11T22:25:51,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/29d5f51210ba4d6f8dfac29a0327dd44 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/29d5f51210ba4d6f8dfac29a0327dd44 2024-11-11T22:25:51,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c5af2a140d024c8593716d5f71a83fbf to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c5af2a140d024c8593716d5f71a83fbf 2024-11-11T22:25:51,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/ed80ea7c5d4043de9414bfb99d6fe219 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/ed80ea7c5d4043de9414bfb99d6fe219 2024-11-11T22:25:51,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/cba64166e777475a9176acc9ad483969 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/cba64166e777475a9176acc9ad483969 2024-11-11T22:25:51,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/05bcdf107cbf4c9d91932e50a884923e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/05bcdf107cbf4c9d91932e50a884923e 2024-11-11T22:25:51,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8e8c9e661906423980d1120b1105acaf to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8e8c9e661906423980d1120b1105acaf 2024-11-11T22:25:51,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e589c5556a1349559380cee995023238 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e589c5556a1349559380cee995023238 2024-11-11T22:25:51,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c65dfb7fa36d41d9a966bd173b61473e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/c65dfb7fa36d41d9a966bd173b61473e 2024-11-11T22:25:51,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/7702e3d74c1b43b686aff98c156a806e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/7702e3d74c1b43b686aff98c156a806e 2024-11-11T22:25:51,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/eac2c030dcce4c728db3ba2ea724a5c8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/eac2c030dcce4c728db3ba2ea724a5c8 2024-11-11T22:25:51,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/41beb201d8f4408eaa00be0ca53fdb9e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/41beb201d8f4408eaa00be0ca53fdb9e 2024-11-11T22:25:51,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/97d9c46de7644111959a58748f7e8f19 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/97d9c46de7644111959a58748f7e8f19 2024-11-11T22:25:51,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/1395214b2ac347d8a325b1ac56c06007 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/1395214b2ac347d8a325b1ac56c06007 2024-11-11T22:25:51,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/185aec4118c847d8825fc5338d72af8f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/185aec4118c847d8825fc5338d72af8f 2024-11-11T22:25:51,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/79e5e9c94e824d148d394324041ba70d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/79e5e9c94e824d148d394324041ba70d 2024-11-11T22:25:51,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/f21d8dffd38c4555a1e67aadc7636dac to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/f21d8dffd38c4555a1e67aadc7636dac 2024-11-11T22:25:51,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/6cea76a3a74a4f2ab996fb95ac294620 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/6cea76a3a74a4f2ab996fb95ac294620 2024-11-11T22:25:51,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e7a840e1a0594912865974b7936146ef to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e7a840e1a0594912865974b7936146ef 2024-11-11T22:25:51,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e9979a5de8594e1ab99e3df7849a39c2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e9979a5de8594e1ab99e3df7849a39c2 2024-11-11T22:25:51,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a393e12f75f346feb609e9cd9a4268eb to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a393e12f75f346feb609e9cd9a4268eb 2024-11-11T22:25:51,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/5497dc9da226405cb84a8644b08f91b4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/5497dc9da226405cb84a8644b08f91b4 2024-11-11T22:25:51,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a4fd4ec444be4a4d9bf9bfaaaa064b5b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/a4fd4ec444be4a4d9bf9bfaaaa064b5b 2024-11-11T22:25:51,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/4871125d5c3f45d0911c3c40e4e705bc to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/4871125d5c3f45d0911c3c40e4e705bc 2024-11-11T22:25:51,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d819d8a70c904d0f93d56386f4556592 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d819d8a70c904d0f93d56386f4556592 2024-11-11T22:25:51,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b029456c8b5e4a61ac8954b5813d4d55 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b029456c8b5e4a61ac8954b5813d4d55 2024-11-11T22:25:51,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/21a88942f2f04081950e865840165405 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/21a88942f2f04081950e865840165405 2024-11-11T22:25:51,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/829496052ed24e6591caeb418f8ab835 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/829496052ed24e6591caeb418f8ab835 2024-11-11T22:25:51,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b3f0bc684e834abc863a119ee1e4c7de to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b3f0bc684e834abc863a119ee1e4c7de 2024-11-11T22:25:51,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8ddc5d41d84a4004b5d02e6b48426658 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/8ddc5d41d84a4004b5d02e6b48426658 2024-11-11T22:25:51,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d65ac6f93b7a41fbbf950ca46b40045b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/d65ac6f93b7a41fbbf950ca46b40045b 2024-11-11T22:25:51,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b070c3c559bd49168b53552a9ab828ce to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/b070c3c559bd49168b53552a9ab828ce 2024-11-11T22:25:51,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/721ea01643df47e7a4174200d6246627 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/721ea01643df47e7a4174200d6246627 2024-11-11T22:25:51,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e8b9ea2ec3b04aa3933a00b4362cbec1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/e8b9ea2ec3b04aa3933a00b4362cbec1 2024-11-11T22:25:51,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/0581566ca31445f1b51c443458ad7edb to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/0581566ca31445f1b51c443458ad7edb 2024-11-11T22:25:51,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/10e6dfa58dfd46db8483ae56eae2138e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/10e6dfa58dfd46db8483ae56eae2138e 2024-11-11T22:25:51,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c3be75bc59814b4ba3d3c501d8e71c06, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/274920d3f8894c89b60a6859d2057f29, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ca348bcabe7b446a8cf5a7c41fc61b44, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/9ea3f7b271e24008b208f07e4797558e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/be6d2a2d40794c97b5c2dfe9a49cadc7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cab2d8da292a4920bc1301f89e2dbeb8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e9a4bd2628fb4307808c41c20083d8f6, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c4b9166c8a6843a6b35dc664f6758209, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f49e885949e5497187afa9dbce4c935e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fe41aa28213a456f9a483788a93bc2a1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/034dad3fe2ff4d15bc700bd7b575510c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/eea258ecdf6f40ee94246667640aa32c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f28e12cebed34aa99ded2de0150f9aab, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c1da72d648244434a753e1ebf7628ce4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/a000d67d37194504904c61ac655d901c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/d3f2c215b07e4c9d9e4bec61e8c31e96, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fd2b044e719b4094afedebb3c20408a3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/00356badc91d4b55b1ca74726b2fe7f7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/b2fde91b611e414dac51b6bd510e9d7c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/92725a4904b244b8a7eec64a39080778, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cf7617cf6f6e43aba4388595c00111d0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ee3ceae0087c4c128efa94526114c05a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/266481dab8024d38aa850f42440a977c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80f96be673f841c49fb463a98925b1fb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e8bfa631053d45e5b42a925d7e8b9d5e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/23ffcfdc85624027b1d123ba46fb65e2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2f1de4ddbaef4fa8a7bd5611fab72b9c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/8cf312868f3c4329949c41f1f1954b2e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/add2f8d1125a401289f52f146722b1e9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80e02b35a5384dd18f4de6aba26d8fa4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2ec9f8c1e30b4009921d02a8a458d98a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/85ad1364099c446ba538e95e12a718d5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/12b1b449e9db493a8a977ad3ba58c032, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/921671bf56254e26b7fa560e79b66b30, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/4b1ee3b18dbe40fa8aae0810c05a3bce, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/65d054d58d3a475bb932e7abc0eb6fb1] to archive 2024-11-11T22:25:51,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:25:51,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c3be75bc59814b4ba3d3c501d8e71c06 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c3be75bc59814b4ba3d3c501d8e71c06 2024-11-11T22:25:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:25:51,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/274920d3f8894c89b60a6859d2057f29 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/274920d3f8894c89b60a6859d2057f29 2024-11-11T22:25:51,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ca348bcabe7b446a8cf5a7c41fc61b44 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ca348bcabe7b446a8cf5a7c41fc61b44 2024-11-11T22:25:51,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/9ea3f7b271e24008b208f07e4797558e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/9ea3f7b271e24008b208f07e4797558e 2024-11-11T22:25:51,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/be6d2a2d40794c97b5c2dfe9a49cadc7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/be6d2a2d40794c97b5c2dfe9a49cadc7 2024-11-11T22:25:51,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cab2d8da292a4920bc1301f89e2dbeb8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cab2d8da292a4920bc1301f89e2dbeb8 2024-11-11T22:25:51,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e9a4bd2628fb4307808c41c20083d8f6 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e9a4bd2628fb4307808c41c20083d8f6 2024-11-11T22:25:51,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c4b9166c8a6843a6b35dc664f6758209 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c4b9166c8a6843a6b35dc664f6758209 2024-11-11T22:25:51,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f49e885949e5497187afa9dbce4c935e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f49e885949e5497187afa9dbce4c935e 2024-11-11T22:25:51,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fe41aa28213a456f9a483788a93bc2a1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fe41aa28213a456f9a483788a93bc2a1 2024-11-11T22:25:51,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/034dad3fe2ff4d15bc700bd7b575510c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/034dad3fe2ff4d15bc700bd7b575510c 2024-11-11T22:25:51,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/eea258ecdf6f40ee94246667640aa32c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/eea258ecdf6f40ee94246667640aa32c 2024-11-11T22:25:51,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f28e12cebed34aa99ded2de0150f9aab to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/f28e12cebed34aa99ded2de0150f9aab 2024-11-11T22:25:52,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c1da72d648244434a753e1ebf7628ce4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/c1da72d648244434a753e1ebf7628ce4 2024-11-11T22:25:52,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/a000d67d37194504904c61ac655d901c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/a000d67d37194504904c61ac655d901c 2024-11-11T22:25:52,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/d3f2c215b07e4c9d9e4bec61e8c31e96 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/d3f2c215b07e4c9d9e4bec61e8c31e96 2024-11-11T22:25:52,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fd2b044e719b4094afedebb3c20408a3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/fd2b044e719b4094afedebb3c20408a3 2024-11-11T22:25:52,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/00356badc91d4b55b1ca74726b2fe7f7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/00356badc91d4b55b1ca74726b2fe7f7 2024-11-11T22:25:52,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/b2fde91b611e414dac51b6bd510e9d7c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/b2fde91b611e414dac51b6bd510e9d7c 2024-11-11T22:25:52,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/92725a4904b244b8a7eec64a39080778 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/92725a4904b244b8a7eec64a39080778 2024-11-11T22:25:52,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cf7617cf6f6e43aba4388595c00111d0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/cf7617cf6f6e43aba4388595c00111d0 2024-11-11T22:25:52,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ee3ceae0087c4c128efa94526114c05a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ee3ceae0087c4c128efa94526114c05a 2024-11-11T22:25:52,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/266481dab8024d38aa850f42440a977c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/266481dab8024d38aa850f42440a977c 2024-11-11T22:25:52,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80f96be673f841c49fb463a98925b1fb to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80f96be673f841c49fb463a98925b1fb 2024-11-11T22:25:52,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e8bfa631053d45e5b42a925d7e8b9d5e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/e8bfa631053d45e5b42a925d7e8b9d5e 2024-11-11T22:25:52,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/23ffcfdc85624027b1d123ba46fb65e2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/23ffcfdc85624027b1d123ba46fb65e2 2024-11-11T22:25:52,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2f1de4ddbaef4fa8a7bd5611fab72b9c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2f1de4ddbaef4fa8a7bd5611fab72b9c 2024-11-11T22:25:52,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/8cf312868f3c4329949c41f1f1954b2e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/8cf312868f3c4329949c41f1f1954b2e 2024-11-11T22:25:52,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/add2f8d1125a401289f52f146722b1e9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/add2f8d1125a401289f52f146722b1e9 2024-11-11T22:25:52,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80e02b35a5384dd18f4de6aba26d8fa4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/80e02b35a5384dd18f4de6aba26d8fa4 2024-11-11T22:25:52,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2ec9f8c1e30b4009921d02a8a458d98a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/2ec9f8c1e30b4009921d02a8a458d98a 2024-11-11T22:25:52,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/85ad1364099c446ba538e95e12a718d5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/85ad1364099c446ba538e95e12a718d5 2024-11-11T22:25:52,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/12b1b449e9db493a8a977ad3ba58c032 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/12b1b449e9db493a8a977ad3ba58c032 2024-11-11T22:25:52,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/921671bf56254e26b7fa560e79b66b30 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/921671bf56254e26b7fa560e79b66b30 2024-11-11T22:25:52,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/4b1ee3b18dbe40fa8aae0810c05a3bce to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/4b1ee3b18dbe40fa8aae0810c05a3bce 2024-11-11T22:25:52,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/65d054d58d3a475bb932e7abc0eb6fb1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/65d054d58d3a475bb932e7abc0eb6fb1 2024-11-11T22:25:52,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/490fe132312744d3ad89d4842967577e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1d98d1298734b699aad6738fc4ab1c0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e25d41164771421989019d14d92bc7e9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f0af9870b7ef4caf8d68d31c8242a819, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/13b44a140e1e4b8e8ab6d5094af7ca22, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/173b164761434d88b5d586b76e675157, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9865bff02fd0499ca179673162be26ea, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4364401ff7ad41d7bad970ef0a004c42, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/389e1a10b7ad47b7a0da3ec103703a2f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/c82edde4ba404dd6ae83ee80a8b3ee84, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/8ff6db10d086472b936c6bafebf33dff, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1a9c1ec18d84014b70a5b1d3e05434f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/69aca1bddbce4be68838a3d11534e351, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/875bcfbd894d43d4a478f434525c96d6, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d9411f2a0f1a424a98b58c6988db7e86, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/af7b526f44cf4757948f93b1fa43b14b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b6f05fcc430748ab8b716fcd0a495272, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9a7cd18c13974585acacbd1659df86d7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/dc56951a950142079a6b2791bda304c7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e42c623298224ce3bd50a71636d2ace3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/7c313a44bf7c48019f2d4341f3e823a7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/0b0aef0687664e44b930c2418560395d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d6d0c35a110a41a49cf15360bdb09f31, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1bae7064f566400f964f53909bab9252, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9051051ac6504bbb87357aaf59fc3b3b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71f1a4d90b69438e836f1f3746bb2fbe, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/5dcd0ab4ea324c1b8c86c777b4f71c65, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/62d66282b57142ea966bc18d8a2c77b5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4a307c724b6c4f809ef3520a63d5def1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71a15af38db142cda372836459723c46, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/6d0d747dacff402fadf5cc78efcc3630, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/3f4045850e674066a44b1186e6369c1b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b3aea0ee98fb45889c744bcb3447cbb6, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/be3e1070198545469785884eb19a40ff, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1cd6365964a840aaae82a33e689ee311, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f8500a5455c84c83921605aa48e996cc] to archive 2024-11-11T22:25:52,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:25:52,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/490fe132312744d3ad89d4842967577e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/490fe132312744d3ad89d4842967577e 2024-11-11T22:25:52,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1d98d1298734b699aad6738fc4ab1c0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1d98d1298734b699aad6738fc4ab1c0 2024-11-11T22:25:52,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e25d41164771421989019d14d92bc7e9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e25d41164771421989019d14d92bc7e9 2024-11-11T22:25:52,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f0af9870b7ef4caf8d68d31c8242a819 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f0af9870b7ef4caf8d68d31c8242a819 2024-11-11T22:25:52,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/13b44a140e1e4b8e8ab6d5094af7ca22 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/13b44a140e1e4b8e8ab6d5094af7ca22 2024-11-11T22:25:52,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/173b164761434d88b5d586b76e675157 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/173b164761434d88b5d586b76e675157 2024-11-11T22:25:52,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9865bff02fd0499ca179673162be26ea to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9865bff02fd0499ca179673162be26ea 2024-11-11T22:25:52,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4364401ff7ad41d7bad970ef0a004c42 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4364401ff7ad41d7bad970ef0a004c42 2024-11-11T22:25:52,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/389e1a10b7ad47b7a0da3ec103703a2f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/389e1a10b7ad47b7a0da3ec103703a2f 2024-11-11T22:25:52,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/c82edde4ba404dd6ae83ee80a8b3ee84 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/c82edde4ba404dd6ae83ee80a8b3ee84 2024-11-11T22:25:52,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/8ff6db10d086472b936c6bafebf33dff to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/8ff6db10d086472b936c6bafebf33dff 2024-11-11T22:25:52,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1a9c1ec18d84014b70a5b1d3e05434f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/a1a9c1ec18d84014b70a5b1d3e05434f 2024-11-11T22:25:52,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/69aca1bddbce4be68838a3d11534e351 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/69aca1bddbce4be68838a3d11534e351 2024-11-11T22:25:52,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/875bcfbd894d43d4a478f434525c96d6 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/875bcfbd894d43d4a478f434525c96d6 2024-11-11T22:25:52,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d9411f2a0f1a424a98b58c6988db7e86 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d9411f2a0f1a424a98b58c6988db7e86 2024-11-11T22:25:52,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/af7b526f44cf4757948f93b1fa43b14b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/af7b526f44cf4757948f93b1fa43b14b 2024-11-11T22:25:52,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b6f05fcc430748ab8b716fcd0a495272 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b6f05fcc430748ab8b716fcd0a495272 2024-11-11T22:25:52,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9a7cd18c13974585acacbd1659df86d7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9a7cd18c13974585acacbd1659df86d7 2024-11-11T22:25:52,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/dc56951a950142079a6b2791bda304c7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/dc56951a950142079a6b2791bda304c7 2024-11-11T22:25:52,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e42c623298224ce3bd50a71636d2ace3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/e42c623298224ce3bd50a71636d2ace3 2024-11-11T22:25:52,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/7c313a44bf7c48019f2d4341f3e823a7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/7c313a44bf7c48019f2d4341f3e823a7 2024-11-11T22:25:52,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/0b0aef0687664e44b930c2418560395d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/0b0aef0687664e44b930c2418560395d 2024-11-11T22:25:52,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d6d0c35a110a41a49cf15360bdb09f31 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/d6d0c35a110a41a49cf15360bdb09f31 2024-11-11T22:25:52,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1bae7064f566400f964f53909bab9252 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1bae7064f566400f964f53909bab9252 2024-11-11T22:25:52,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9051051ac6504bbb87357aaf59fc3b3b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9051051ac6504bbb87357aaf59fc3b3b 2024-11-11T22:25:52,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71f1a4d90b69438e836f1f3746bb2fbe to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71f1a4d90b69438e836f1f3746bb2fbe 2024-11-11T22:25:52,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/5dcd0ab4ea324c1b8c86c777b4f71c65 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/5dcd0ab4ea324c1b8c86c777b4f71c65 2024-11-11T22:25:52,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/62d66282b57142ea966bc18d8a2c77b5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/62d66282b57142ea966bc18d8a2c77b5 2024-11-11T22:25:52,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4a307c724b6c4f809ef3520a63d5def1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/4a307c724b6c4f809ef3520a63d5def1 2024-11-11T22:25:52,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71a15af38db142cda372836459723c46 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/71a15af38db142cda372836459723c46 2024-11-11T22:25:52,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/6d0d747dacff402fadf5cc78efcc3630 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/6d0d747dacff402fadf5cc78efcc3630 2024-11-11T22:25:52,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/3f4045850e674066a44b1186e6369c1b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/3f4045850e674066a44b1186e6369c1b 2024-11-11T22:25:52,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b3aea0ee98fb45889c744bcb3447cbb6 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b3aea0ee98fb45889c744bcb3447cbb6 2024-11-11T22:25:52,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/be3e1070198545469785884eb19a40ff to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/be3e1070198545469785884eb19a40ff 2024-11-11T22:25:52,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1cd6365964a840aaae82a33e689ee311 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/1cd6365964a840aaae82a33e689ee311 2024-11-11T22:25:52,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f8500a5455c84c83921605aa48e996cc to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/f8500a5455c84c83921605aa48e996cc 2024-11-11T22:25:52,125 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/recovered.edits/539.seqid, newMaxSeqId=539, maxSeqId=1 2024-11-11T22:25:52,126 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. 2024-11-11T22:25:52,126 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 765de8aa1edd426d207e09298c569dc6: Waiting for close lock at 1731363951012Running coprocessor pre-close hooks at 1731363951012Disabling compacts and flushes for region at 1731363951012Disabling writes for close at 1731363951012Obtaining lock to block concurrent updates at 1731363951012Preparing flush snapshotting stores in 765de8aa1edd426d207e09298c569dc6 at 1731363951012Finished memstore snapshotting TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731363951012Flushing stores of TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6. at 1731363951013 (+1 ms)Flushing 765de8aa1edd426d207e09298c569dc6/A: creating writer at 1731363951013Flushing 765de8aa1edd426d207e09298c569dc6/A: appending metadata at 1731363951017 (+4 ms)Flushing 765de8aa1edd426d207e09298c569dc6/A: closing flushed file at 1731363951017Flushing 765de8aa1edd426d207e09298c569dc6/B: creating writer at 1731363951433 (+416 ms)Flushing 765de8aa1edd426d207e09298c569dc6/B: appending metadata at 1731363951437 (+4 ms)Flushing 765de8aa1edd426d207e09298c569dc6/B: closing flushed file at 1731363951437Flushing 765de8aa1edd426d207e09298c569dc6/C: creating writer at 1731363951452 (+15 ms)Flushing 765de8aa1edd426d207e09298c569dc6/C: appending metadata at 1731363951455 (+3 ms)Flushing 765de8aa1edd426d207e09298c569dc6/C: closing flushed file at 1731363951455Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22849946: reopening flushed file at 1731363951869 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bb4cd74: reopening flushed file at 1731363951875 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ec67957: reopening flushed file at 1731363951882 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 765de8aa1edd426d207e09298c569dc6 in 876ms, sequenceid=536, compaction requested=true at 1731363951888 (+6 ms)Writing region close event to WAL at 1731363952121 (+233 ms)Running coprocessor post-close hooks at 1731363952126 (+5 ms)Closed at 1731363952126 2024-11-11T22:25:52,129 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:52,130 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=765de8aa1edd426d207e09298c569dc6, regionState=CLOSED 2024-11-11T22:25:52,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:25:52,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T22:25:52,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 765de8aa1edd426d207e09298c569dc6, server=8d82b89dd27d,39613,1731363918313 in 1.2800 sec 2024-11-11T22:25:52,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-11T22:25:52,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=765de8aa1edd426d207e09298c569dc6, UNASSIGN in 1.2890 sec 2024-11-11T22:25:52,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T22:25:52,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2980 sec 2024-11-11T22:25:52,143 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363952143"}]},"ts":"1731363952143"} 2024-11-11T22:25:52,146 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:25:52,146 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:25:52,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3160 sec 2024-11-11T22:25:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:25:52,983 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:52,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:25:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:25:52,991 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:25:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T22:25:52,993 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:25:52,997 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:53,002 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/recovered.edits] 2024-11-11T22:25:53,005 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/3104cae5af5d48d19adfc60f29bdd15f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/3104cae5af5d48d19adfc60f29bdd15f 2024-11-11T22:25:53,006 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/5000eff09aec47ce9c0c17e26cce461d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/5000eff09aec47ce9c0c17e26cce461d 2024-11-11T22:25:53,007 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/affd14b5a9b64ebaaef08e07a072175f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/A/affd14b5a9b64ebaaef08e07a072175f 2024-11-11T22:25:53,010 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/9a9fe03a820e484a8ba24f3fe5fa3378 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/9a9fe03a820e484a8ba24f3fe5fa3378 2024-11-11T22:25:53,011 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/a5baadf168bd459a97d9f49559bed3ea to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/a5baadf168bd459a97d9f49559bed3ea 2024-11-11T22:25:53,012 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ccd576e7c0b344dc8c5bbfc10b3eac8c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/B/ccd576e7c0b344dc8c5bbfc10b3eac8c 2024-11-11T22:25:53,014 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9a5f55273d8a4152b22834bdb5e35320 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/9a5f55273d8a4152b22834bdb5e35320 2024-11-11T22:25:53,016 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b124ccf800244b2db3f391947f8c5bac to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/b124ccf800244b2db3f391947f8c5bac 2024-11-11T22:25:53,018 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/ca90c023f74b4c4ab2082f4c5a6943a3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/C/ca90c023f74b4c4ab2082f4c5a6943a3 2024-11-11T22:25:53,022 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/recovered.edits/539.seqid to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6/recovered.edits/539.seqid 2024-11-11T22:25:53,023 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/765de8aa1edd426d207e09298c569dc6 2024-11-11T22:25:53,023 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:25:53,031 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:25:53,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T22:25:53,042 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:25:53,077 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:25:53,078 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:25:53,079 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:25:53,079 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731363953079"}]},"ts":"9223372036854775807"} 2024-11-11T22:25:53,084 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:25:53,084 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 765de8aa1edd426d207e09298c569dc6, NAME => 'TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:25:53,084 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:25:53,084 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731363953084"}]},"ts":"9223372036854775807"} 2024-11-11T22:25:53,088 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:25:53,089 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:25:53,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 102 msec 2024-11-11T22:25:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T22:25:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:25:53,104 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:53,114 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;8d82b89dd27d:39613-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_437819696_22 at /127.0.0.1:50686 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=579 (was 602), ProcessCount=11 (was 11), AvailableMemoryMB=6513 (was 7032) 2024-11-11T22:25:53,122 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=579, ProcessCount=11, AvailableMemoryMB=6511 2024-11-11T22:25:53,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:25:53,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:25:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:25:53,126 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:25:53,126 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:53,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-11T22:25:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:25:53,128 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:25:53,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741954_1130 (size=959) 2024-11-11T22:25:53,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:25:53,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:25:53,536 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e 2024-11-11T22:25:53,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741955_1131 (size=53) 2024-11-11T22:25:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:25:53,943 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:25:53,944 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e47b46921ed127eb101b947b07da0745, disabling compactions & flushes 2024-11-11T22:25:53,944 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:53,944 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:53,944 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. after waiting 0 ms 2024-11-11T22:25:53,944 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:53,944 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:53,944 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e47b46921ed127eb101b947b07da0745: Waiting for close lock at 1731363953944Disabling compacts and flushes for region at 1731363953944Disabling writes for close at 1731363953944Writing region close event to WAL at 1731363953944Closed at 1731363953944 2024-11-11T22:25:53,945 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:25:53,946 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731363953945"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731363953945"}]},"ts":"1731363953945"} 2024-11-11T22:25:53,947 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:25:53,948 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:25:53,949 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363953949"}]},"ts":"1731363953949"} 2024-11-11T22:25:53,951 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:25:53,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, ASSIGN}] 2024-11-11T22:25:53,952 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, ASSIGN 2024-11-11T22:25:53,953 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, ASSIGN; state=OFFLINE, location=8d82b89dd27d,39613,1731363918313; forceNewPlan=false, retain=false 2024-11-11T22:25:54,104 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e47b46921ed127eb101b947b07da0745, regionState=OPENING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:54,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, ASSIGN because future has completed 2024-11-11T22:25:54,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:25:54,262 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:54,262 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:25:54,262 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,262 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:25:54,262 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,262 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:25:54,264 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,266 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e47b46921ed127eb101b947b07da0745 columnFamilyName A 2024-11-11T22:25:54,266 DEBUG [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:54,267 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(327): Store=e47b46921ed127eb101b947b07da0745/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:54,267 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,268 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e47b46921ed127eb101b947b07da0745 columnFamilyName B 2024-11-11T22:25:54,268 DEBUG [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:54,269 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(327): Store=e47b46921ed127eb101b947b07da0745/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:54,269 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,270 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e47b46921ed127eb101b947b07da0745 columnFamilyName C 2024-11-11T22:25:54,270 DEBUG [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:54,270 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(327): Store=e47b46921ed127eb101b947b07da0745/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:54,271 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,271 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,272 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,272 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,273 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,273 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:25:54,274 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,276 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:25:54,276 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened e47b46921ed127eb101b947b07da0745; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58906414, jitterRate=-0.12222602963447571}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:25:54,276 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:54,277 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for e47b46921ed127eb101b947b07da0745: Running coprocessor pre-open hook at 1731363954263Writing region info on filesystem at 1731363954263Initializing all the Stores at 1731363954263Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363954264 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363954264Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363954264Cleaning up temporary data from old regions at 1731363954273 (+9 ms)Running coprocessor post-open hooks at 1731363954276 (+3 ms)Region opened successfully at 1731363954277 (+1 ms) 2024-11-11T22:25:54,278 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., pid=36, masterSystemTime=1731363954258 2024-11-11T22:25:54,280 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:54,280 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:54,281 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e47b46921ed127eb101b947b07da0745, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:54,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:25:54,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-11T22:25:54,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 in 178 msec 2024-11-11T22:25:54,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-11T22:25:54,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, ASSIGN in 334 msec 2024-11-11T22:25:54,288 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:25:54,288 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363954288"}]},"ts":"1731363954288"} 2024-11-11T22:25:54,290 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:25:54,291 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:25:54,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-11T22:25:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:25:55,274 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:55,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:55,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:55,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:55,280 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e595508, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:55,284 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b05381, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:55,286 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:55,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:55,292 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:25:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,40133,1731363917560 2024-11-11T22:25:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14bf5f07 2024-11-11T22:25:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:25:55,294 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:25:55,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:25:55,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:25:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T22:25:55,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741956_1132 (size=995) 2024-11-11T22:25:55,722 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T22:25:55,722 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T22:25:55,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:25:55,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, REOPEN/MOVE}] 2024-11-11T22:25:55,732 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, REOPEN/MOVE 2024-11-11T22:25:55,733 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e47b46921ed127eb101b947b07da0745, regionState=CLOSING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:55,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, REOPEN/MOVE because future has completed 2024-11-11T22:25:55,735 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:25:55,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:25:55,889 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:55,890 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:25:55,890 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing e47b46921ed127eb101b947b07da0745, disabling compactions & flushes 2024-11-11T22:25:55,890 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:55,890 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:55,890 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. after waiting 0 ms 2024-11-11T22:25:55,890 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:55,899 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T22:25:55,900 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:55,900 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for e47b46921ed127eb101b947b07da0745: Waiting for close lock at 1731363955890Running coprocessor pre-close hooks at 1731363955890Disabling compacts and flushes for region at 1731363955890Disabling writes for close at 1731363955890Writing region close event to WAL at 1731363955891 (+1 ms)Running coprocessor post-close hooks at 1731363955900 (+9 ms)Closed at 1731363955900 2024-11-11T22:25:55,900 WARN [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: e47b46921ed127eb101b947b07da0745 to self. 2024-11-11T22:25:55,903 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:55,904 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e47b46921ed127eb101b947b07da0745, regionState=CLOSED 2024-11-11T22:25:55,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:25:55,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-11T22:25:55,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 in 172 msec 2024-11-11T22:25:55,910 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, REOPEN/MOVE; state=CLOSED, location=8d82b89dd27d,39613,1731363918313; forceNewPlan=false, retain=true 2024-11-11T22:25:56,061 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e47b46921ed127eb101b947b07da0745, regionState=OPENING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, REOPEN/MOVE because future has completed 2024-11-11T22:25:56,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:25:56,224 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,224 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:25:56,224 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,225 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:25:56,225 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,225 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,227 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,231 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e47b46921ed127eb101b947b07da0745 columnFamilyName A 2024-11-11T22:25:56,233 DEBUG [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:56,234 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(327): Store=e47b46921ed127eb101b947b07da0745/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:56,234 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,235 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e47b46921ed127eb101b947b07da0745 columnFamilyName B 2024-11-11T22:25:56,235 DEBUG [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:56,235 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(327): Store=e47b46921ed127eb101b947b07da0745/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:56,235 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,236 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e47b46921ed127eb101b947b07da0745 columnFamilyName C 2024-11-11T22:25:56,236 DEBUG [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:56,237 INFO [StoreOpener-e47b46921ed127eb101b947b07da0745-1 {}] regionserver.HStore(327): Store=e47b46921ed127eb101b947b07da0745/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:25:56,237 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,237 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,239 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,240 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,240 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,240 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:25:56,242 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,242 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened e47b46921ed127eb101b947b07da0745; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66477719, jitterRate=-0.009404793381690979}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:25:56,243 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,244 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for e47b46921ed127eb101b947b07da0745: Running coprocessor pre-open hook at 1731363956225Writing region info on filesystem at 1731363956225Initializing all the Stores at 1731363956225Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363956226 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363956226Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363956226Cleaning up temporary data from old regions at 1731363956240 (+14 ms)Running coprocessor post-open hooks at 1731363956243 (+3 ms)Region opened successfully at 1731363956244 (+1 ms) 2024-11-11T22:25:56,245 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., pid=41, masterSystemTime=1731363956221 2024-11-11T22:25:56,248 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,249 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,249 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e47b46921ed127eb101b947b07da0745, regionState=OPEN, openSeqNum=5, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:25:56,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-11T22:25:56,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 in 187 msec 2024-11-11T22:25:56,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-11T22:25:56,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, REOPEN/MOVE in 522 msec 2024-11-11T22:25:56,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-11T22:25:56,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-11-11T22:25:56,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 957 msec 2024-11-11T22:25:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-11T22:25:56,264 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:25:56,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:56,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:56,267 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e76ac5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:56,269 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604a2ea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,270 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:56,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:56,271 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f34c532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:56,273 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6055b0fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,275 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:56,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:56,276 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:56,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c8d80f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:56,278 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553606c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:56,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:56,281 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0ea7bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:56,284 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13fdaefb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:56,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:56,286 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce1796e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:56,288 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ecb63cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:56,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:56,291 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205584d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:56,293 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1977df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:56,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:56,295 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293659ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:56,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:56,297 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efac9a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:56,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:56,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c62e8f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:56,302 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c27714a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:25:56,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:25:56,305 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:25:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:25:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:25:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c679eee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:25:56,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:25:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c558fb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:25:56,309 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:56,309 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:56,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:25:56,309 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:56,310 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:56,309 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T22:25:56,310 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:56,312 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:25:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:56,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:56,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:56,314 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:25:56,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:25:56,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:25:56,314 DEBUG [Thread-668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:56,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:56,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:56,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:25:56,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:25:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:25:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:25:56,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:25:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:25:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:25:56,335 DEBUG [Thread-666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:56,335 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:25:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:25:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:25:56,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:56,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:25:56,340 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:56,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:25:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:25:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:25:56,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111156a95a6f78164797a5c6c6a9e63f1624_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363956339/Put/seqid=0 2024-11-11T22:25:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:25:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364016415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364016417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364016420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364016426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364016430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741957_1133 (size=12154) 2024-11-11T22:25:56,453 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:56,466 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111156a95a6f78164797a5c6c6a9e63f1624_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111156a95a6f78164797a5c6c6a9e63f1624_e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,469 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/139717c38a7b4f18bbce35d55d29bbd0, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:25:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/139717c38a7b4f18bbce35d55d29bbd0 is 175, key is test_row_0/A:col10/1731363956339/Put/seqid=0 2024-11-11T22:25:56,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741958_1134 (size=30955) 2024-11-11T22:25:56,534 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/139717c38a7b4f18bbce35d55d29bbd0 2024-11-11T22:25:56,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364016533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364016534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364016534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364016543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364016543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/226d32248ee2447d99a1a13ad5024863 is 50, key is test_row_0/B:col10/1731363956339/Put/seqid=0 2024-11-11T22:25:56,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741959_1135 (size=12001) 2024-11-11T22:25:56,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/226d32248ee2447d99a1a13ad5024863 2024-11-11T22:25:56,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:25:56,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/1f2b500d1f8d490ca202d8fad1f4e5c3 is 50, key is test_row_0/C:col10/1731363956339/Put/seqid=0 2024-11-11T22:25:56,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741960_1136 (size=12001) 2024-11-11T22:25:56,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/1f2b500d1f8d490ca202d8fad1f4e5c3 2024-11-11T22:25:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/139717c38a7b4f18bbce35d55d29bbd0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/139717c38a7b4f18bbce35d55d29bbd0 2024-11-11T22:25:56,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/139717c38a7b4f18bbce35d55d29bbd0, entries=150, sequenceid=15, filesize=30.2 K 2024-11-11T22:25:56,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/226d32248ee2447d99a1a13ad5024863 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/226d32248ee2447d99a1a13ad5024863 2024-11-11T22:25:56,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/226d32248ee2447d99a1a13ad5024863, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T22:25:56,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/1f2b500d1f8d490ca202d8fad1f4e5c3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/1f2b500d1f8d490ca202d8fad1f4e5c3 2024-11-11T22:25:56,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364016743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/1f2b500d1f8d490ca202d8fad1f4e5c3, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T22:25:56,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e47b46921ed127eb101b947b07da0745 in 410ms, sequenceid=15, compaction requested=false 2024-11-11T22:25:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:25:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:56,754 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:56,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T22:25:56,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:56,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:56,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ebdcbc1554394ade9985b134af72e06e_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363956411/Put/seqid=0 2024-11-11T22:25:56,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364016776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364016773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741961_1137 (size=17034) 2024-11-11T22:25:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364016793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364016793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364016904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364016904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364016913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364016913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:56,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:56,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:56,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:56,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:25:57,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364017055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:57,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:57,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364017125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364017125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364017125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364017141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:57,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ebdcbc1554394ade9985b134af72e06e_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ebdcbc1554394ade9985b134af72e06e_e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:57,212 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/f3f6448e721f49b9b49a8f0ac8b01f96, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:25:57,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/f3f6448e721f49b9b49a8f0ac8b01f96 is 175, key is test_row_0/A:col10/1731363956411/Put/seqid=0 2024-11-11T22:25:57,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741962_1138 (size=48139) 2024-11-11T22:25:57,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:57,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:57,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364017434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364017434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364017434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:25:57,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364017463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364017573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,675 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/f3f6448e721f49b9b49a8f0ac8b01f96 2024-11-11T22:25:57,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/d1386f320408416ba13b3eb42c8b7b0a is 50, key is test_row_0/B:col10/1731363956411/Put/seqid=0 2024-11-11T22:25:57,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741963_1139 (size=12001) 2024-11-11T22:25:57,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:57,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,728 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:25:57,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:57,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:57,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:57,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:57,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364017943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364017953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364017953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:57,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364017975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:58,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:58,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:58,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/d1386f320408416ba13b3eb42c8b7b0a 2024-11-11T22:25:58,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/9cbbf887512d44d6872083b9b3c13e39 is 50, key is test_row_0/C:col10/1731363956411/Put/seqid=0 2024-11-11T22:25:58,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741964_1140 (size=12001) 2024-11-11T22:25:58,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/9cbbf887512d44d6872083b9b3c13e39 2024-11-11T22:25:58,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/f3f6448e721f49b9b49a8f0ac8b01f96 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/f3f6448e721f49b9b49a8f0ac8b01f96 2024-11-11T22:25:58,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:58,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:25:58,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/f3f6448e721f49b9b49a8f0ac8b01f96, entries=250, sequenceid=42, filesize=47.0 K 2024-11-11T22:25:58,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/d1386f320408416ba13b3eb42c8b7b0a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d1386f320408416ba13b3eb42c8b7b0a 2024-11-11T22:25:58,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d1386f320408416ba13b3eb42c8b7b0a, entries=150, sequenceid=42, filesize=11.7 K 2024-11-11T22:25:58,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/9cbbf887512d44d6872083b9b3c13e39 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9cbbf887512d44d6872083b9b3c13e39 2024-11-11T22:25:58,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9cbbf887512d44d6872083b9b3c13e39, entries=150, sequenceid=42, filesize=11.7 K 2024-11-11T22:25:58,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for e47b46921ed127eb101b947b07da0745 in 1433ms, sequenceid=42, compaction requested=false 2024-11-11T22:25:58,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:25:58,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:25:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:58,320 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-11T22:25:58,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111995181acc25d4433a0b13090cd1afa98_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363956788/Put/seqid=0 2024-11-11T22:25:58,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741965_1141 (size=12154) 2024-11-11T22:25:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:25:58,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:25:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:58,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364018694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:58,744 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111995181acc25d4433a0b13090cd1afa98_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111995181acc25d4433a0b13090cd1afa98_e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:58,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/5378be40d5854d0cb262269a65441c48, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:25:58,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/5378be40d5854d0cb262269a65441c48 is 175, key is test_row_0/A:col10/1731363956788/Put/seqid=0 2024-11-11T22:25:58,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741966_1142 (size=30955) 2024-11-11T22:25:58,762 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/5378be40d5854d0cb262269a65441c48 2024-11-11T22:25:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/fc23dc4d27414267b3a0a9254b868c5a is 50, key is test_row_0/B:col10/1731363956788/Put/seqid=0 2024-11-11T22:25:58,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741967_1143 (size=12001) 2024-11-11T22:25:58,795 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/fc23dc4d27414267b3a0a9254b868c5a 2024-11-11T22:25:58,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364018803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:58,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/3688e6fad17c4589b1f03e850db94c72 is 50, key is test_row_0/C:col10/1731363956788/Put/seqid=0 2024-11-11T22:25:58,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741968_1144 (size=12001) 2024-11-11T22:25:58,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364018953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:58,958 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:58,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364018964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:58,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364018964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:58,967 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:58,969 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:58,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:58,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364018995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:58,999 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:25:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:59,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364019013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:59,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:59,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:59,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:59,214 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/3688e6fad17c4589b1f03e850db94c72 2024-11-11T22:25:59,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/5378be40d5854d0cb262269a65441c48 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/5378be40d5854d0cb262269a65441c48 2024-11-11T22:25:59,224 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/5378be40d5854d0cb262269a65441c48, entries=150, sequenceid=52, filesize=30.2 K 2024-11-11T22:25:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/fc23dc4d27414267b3a0a9254b868c5a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/fc23dc4d27414267b3a0a9254b868c5a 2024-11-11T22:25:59,229 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/fc23dc4d27414267b3a0a9254b868c5a, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T22:25:59,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/3688e6fad17c4589b1f03e850db94c72 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/3688e6fad17c4589b1f03e850db94c72 2024-11-11T22:25:59,235 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/3688e6fad17c4589b1f03e850db94c72, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T22:25:59,237 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e47b46921ed127eb101b947b07da0745 in 917ms, sequenceid=52, compaction requested=true 2024-11-11T22:25:59,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:25:59,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:25:59,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T22:25:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T22:25:59,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T22:25:59,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9250 sec 2024-11-11T22:25:59,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.9320 sec 2024-11-11T22:25:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:59,325 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:25:59,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T22:25:59,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111acdd6d7d1a2f48ca9007c7ea76cba0da_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363958688/Put/seqid=0 2024-11-11T22:25:59,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741969_1145 (size=14594) 2024-11-11T22:25:59,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364019359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:59,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364019473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364019683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:25:59,747 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:25:59,752 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111acdd6d7d1a2f48ca9007c7ea76cba0da_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111acdd6d7d1a2f48ca9007c7ea76cba0da_e47b46921ed127eb101b947b07da0745 2024-11-11T22:25:59,753 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/492d686db30d499bb7c808462de542c9, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:25:59,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/492d686db30d499bb7c808462de542c9 is 175, key is test_row_0/A:col10/1731363958688/Put/seqid=0 2024-11-11T22:25:59,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741970_1146 (size=39549) 2024-11-11T22:25:59,770 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/492d686db30d499bb7c808462de542c9 2024-11-11T22:25:59,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/48171f62e12643d2acbaafc1d2f84202 is 50, key is test_row_0/B:col10/1731363958688/Put/seqid=0 2024-11-11T22:25:59,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741971_1147 (size=12001) 2024-11-11T22:25:59,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:25:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364019993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:25:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:25:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:00,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/48171f62e12643d2acbaafc1d2f84202 2024-11-11T22:26:00,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/91207489620b4e69bc9d8290b1261eb5 is 50, key is test_row_0/C:col10/1731363958688/Put/seqid=0 2024-11-11T22:26:00,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741972_1148 (size=12001) 2024-11-11T22:26:00,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/91207489620b4e69bc9d8290b1261eb5 2024-11-11T22:26:00,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/492d686db30d499bb7c808462de542c9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/492d686db30d499bb7c808462de542c9 2024-11-11T22:26:00,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/492d686db30d499bb7c808462de542c9, entries=200, sequenceid=79, filesize=38.6 K 2024-11-11T22:26:00,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/48171f62e12643d2acbaafc1d2f84202 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/48171f62e12643d2acbaafc1d2f84202 2024-11-11T22:26:00,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/48171f62e12643d2acbaafc1d2f84202, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T22:26:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/91207489620b4e69bc9d8290b1261eb5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/91207489620b4e69bc9d8290b1261eb5 2024-11-11T22:26:00,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/91207489620b4e69bc9d8290b1261eb5, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T22:26:00,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e47b46921ed127eb101b947b07da0745 in 980ms, sequenceid=79, compaction requested=true 2024-11-11T22:26:00,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:00,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:00,306 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:00,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:00,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:00,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:00,306 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:00,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:00,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:00,307 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:00,307 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/A is initiating minor compaction (all files) 2024-11-11T22:26:00,308 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/A in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:00,308 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/139717c38a7b4f18bbce35d55d29bbd0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/f3f6448e721f49b9b49a8f0ac8b01f96, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/5378be40d5854d0cb262269a65441c48, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/492d686db30d499bb7c808462de542c9] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=146.1 K 2024-11-11T22:26:00,308 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:00,308 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/B is initiating minor compaction (all files) 2024-11-11T22:26:00,308 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/B in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:00,308 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/226d32248ee2447d99a1a13ad5024863, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d1386f320408416ba13b3eb42c8b7b0a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/fc23dc4d27414267b3a0a9254b868c5a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/48171f62e12643d2acbaafc1d2f84202] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=46.9 K 2024-11-11T22:26:00,308 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:00,309 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/139717c38a7b4f18bbce35d55d29bbd0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/f3f6448e721f49b9b49a8f0ac8b01f96, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/5378be40d5854d0cb262269a65441c48, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/492d686db30d499bb7c808462de542c9] 2024-11-11T22:26:00,309 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 226d32248ee2447d99a1a13ad5024863, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731363956331 2024-11-11T22:26:00,309 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 139717c38a7b4f18bbce35d55d29bbd0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731363956331 2024-11-11T22:26:00,310 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting d1386f320408416ba13b3eb42c8b7b0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731363956411 2024-11-11T22:26:00,310 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3f6448e721f49b9b49a8f0ac8b01f96, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731363956411 2024-11-11T22:26:00,310 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting fc23dc4d27414267b3a0a9254b868c5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731363956765 2024-11-11T22:26:00,310 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5378be40d5854d0cb262269a65441c48, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731363956765 2024-11-11T22:26:00,311 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 48171f62e12643d2acbaafc1d2f84202, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363958665 2024-11-11T22:26:00,311 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492d686db30d499bb7c808462de542c9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363958665 2024-11-11T22:26:00,339 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:00,345 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/e6a6d5f9883d43bebea545323b665c02 is 50, key is test_row_0/B:col10/1731363958688/Put/seqid=0 2024-11-11T22:26:00,348 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:00,365 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111137977dd209d54475881c949be4a7f0e8_e47b46921ed127eb101b947b07da0745 store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:00,370 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111137977dd209d54475881c949be4a7f0e8_e47b46921ed127eb101b947b07da0745, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:00,370 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111137977dd209d54475881c949be4a7f0e8_e47b46921ed127eb101b947b07da0745 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:00,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741973_1149 (size=12139) 2024-11-11T22:26:00,395 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/e6a6d5f9883d43bebea545323b665c02 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/e6a6d5f9883d43bebea545323b665c02 2024-11-11T22:26:00,416 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e47b46921ed127eb101b947b07da0745/B of e47b46921ed127eb101b947b07da0745 into e6a6d5f9883d43bebea545323b665c02(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:00,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:00,417 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/B, priority=12, startTime=1731363960306; duration=0sec 2024-11-11T22:26:00,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:00,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:B 2024-11-11T22:26:00,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:00,419 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:00,419 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/C is initiating minor compaction (all files) 2024-11-11T22:26:00,419 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/C in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:00,420 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/1f2b500d1f8d490ca202d8fad1f4e5c3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9cbbf887512d44d6872083b9b3c13e39, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/3688e6fad17c4589b1f03e850db94c72, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/91207489620b4e69bc9d8290b1261eb5] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=46.9 K 2024-11-11T22:26:00,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741974_1150 (size=4469) 2024-11-11T22:26:00,421 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f2b500d1f8d490ca202d8fad1f4e5c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731363956331 2024-11-11T22:26:00,425 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cbbf887512d44d6872083b9b3c13e39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731363956411 2024-11-11T22:26:00,426 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 3688e6fad17c4589b1f03e850db94c72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731363956765 2024-11-11T22:26:00,427 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 91207489620b4e69bc9d8290b1261eb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363958665 2024-11-11T22:26:00,463 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:00,464 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/4719d4b17565458f94f22649d54843ff is 50, key is test_row_0/C:col10/1731363958688/Put/seqid=0 2024-11-11T22:26:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:26:00,484 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:00,484 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T22:26:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:26:00,489 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:00,491 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:00,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:00,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741975_1151 (size=12139) 2024-11-11T22:26:00,545 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/4719d4b17565458f94f22649d54843ff as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4719d4b17565458f94f22649d54843ff 2024-11-11T22:26:00,549 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:00,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:26:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:00,552 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e47b46921ed127eb101b947b07da0745/C of e47b46921ed127eb101b947b07da0745 into 4719d4b17565458f94f22649d54843ff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:00,552 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:00,552 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/C, priority=12, startTime=1731363960306; duration=0sec 2024-11-11T22:26:00,552 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:00,552 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:C 2024-11-11T22:26:00,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a7268523b7514b57b4938b8f317b7630_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363959330/Put/seqid=0 2024-11-11T22:26:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:26:00,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741976_1152 (size=14594) 2024-11-11T22:26:00,615 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:00,624 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a7268523b7514b57b4938b8f317b7630_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a7268523b7514b57b4938b8f317b7630_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:00,625 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8f48727d119645858732205bac784b9d, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:00,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8f48727d119645858732205bac784b9d is 175, key is test_row_0/A:col10/1731363959330/Put/seqid=0 2024-11-11T22:26:00,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741977_1153 (size=39549) 2024-11-11T22:26:00,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:26:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:00,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:00,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:00,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:00,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364020693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:00,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:26:00,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:00,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:00,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:00,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:00,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:26:00,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364020812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:00,822 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#A#compaction#130 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:00,822 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/1089999c69fa47ad9ead07e174838ea8 is 175, key is test_row_0/A:col10/1731363958688/Put/seqid=0 2024-11-11T22:26:00,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741978_1154 (size=31093) 2024-11-11T22:26:00,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:26:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:00,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:00,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364020963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:00,964 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:00,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364020983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:00,987 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:00,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364020993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:01,001 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:01,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364021023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:01,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364021023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:01,026 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:01,047 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8f48727d119645858732205bac784b9d 2024-11-11T22:26:01,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/0d2b5c2579654975874ceeb8cae0dde5 is 50, key is test_row_0/B:col10/1731363959330/Put/seqid=0 2024-11-11T22:26:01,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741979_1155 (size=12001) 2024-11-11T22:26:01,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:26:01,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:01,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:01,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:01,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:01,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:26:01,236 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/1089999c69fa47ad9ead07e174838ea8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/1089999c69fa47ad9ead07e174838ea8 2024-11-11T22:26:01,244 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e47b46921ed127eb101b947b07da0745/A of e47b46921ed127eb101b947b07da0745 into 1089999c69fa47ad9ead07e174838ea8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:01,244 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:01,244 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/A, priority=12, startTime=1731363960306; duration=0sec 2024-11-11T22:26:01,244 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:01,244 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:A 2024-11-11T22:26:01,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:26:01,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:01,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:01,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:01,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:01,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:01,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364021333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:01,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:26:01,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:01,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:01,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:01,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:01,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:01,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/0d2b5c2579654975874ceeb8cae0dde5 2024-11-11T22:26:01,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/54c7faf9001846578658c0367579d019 is 50, key is test_row_0/C:col10/1731363959330/Put/seqid=0 2024-11-11T22:26:01,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741980_1156 (size=12001) 2024-11-11T22:26:01,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/54c7faf9001846578658c0367579d019 2024-11-11T22:26:01,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8f48727d119645858732205bac784b9d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8f48727d119645858732205bac784b9d 2024-11-11T22:26:01,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8f48727d119645858732205bac784b9d, entries=200, sequenceid=91, filesize=38.6 K 2024-11-11T22:26:01,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/0d2b5c2579654975874ceeb8cae0dde5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0d2b5c2579654975874ceeb8cae0dde5 2024-11-11T22:26:01,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0d2b5c2579654975874ceeb8cae0dde5, entries=150, sequenceid=91, filesize=11.7 K 2024-11-11T22:26:01,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/54c7faf9001846578658c0367579d019 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/54c7faf9001846578658c0367579d019 2024-11-11T22:26:01,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/54c7faf9001846578658c0367579d019, entries=150, sequenceid=91, filesize=11.7 K 2024-11-11T22:26:01,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e47b46921ed127eb101b947b07da0745 in 1016ms, sequenceid=91, compaction requested=false 2024-11-11T22:26:01,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:01,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:26:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:01,573 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:26:01,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e0c9ad391801463884f275a71058e914_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363960692/Put/seqid=0 2024-11-11T22:26:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741981_1157 (size=12154) 2024-11-11T22:26:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:26:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:01,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:01,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364021875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:01,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364021983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:01,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:02,002 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e0c9ad391801463884f275a71058e914_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0c9ad391801463884f275a71058e914_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:02,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/489af0702b55442091a0b58507f38489, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:02,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/489af0702b55442091a0b58507f38489 is 175, key is test_row_0/A:col10/1731363960692/Put/seqid=0 2024-11-11T22:26:02,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741982_1158 (size=30955) 2024-11-11T22:26:02,011 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/489af0702b55442091a0b58507f38489 2024-11-11T22:26:02,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/8a969df0952f434e9f35bbdf02facdd8 is 50, key is test_row_0/B:col10/1731363960692/Put/seqid=0 2024-11-11T22:26:02,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741983_1159 (size=12001) 2024-11-11T22:26:02,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364022203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:02,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:02,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:02,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:02,447 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/8a969df0952f434e9f35bbdf02facdd8 2024-11-11T22:26:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/163e7c53019a4e05a101900d7adabd2f is 50, key is test_row_0/C:col10/1731363960692/Put/seqid=0 2024-11-11T22:26:02,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741984_1160 (size=12001) 2024-11-11T22:26:02,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364022513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:26:02,891 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/163e7c53019a4e05a101900d7adabd2f 2024-11-11T22:26:02,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/489af0702b55442091a0b58507f38489 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/489af0702b55442091a0b58507f38489 2024-11-11T22:26:02,914 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/489af0702b55442091a0b58507f38489, entries=150, sequenceid=118, filesize=30.2 K 2024-11-11T22:26:02,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/8a969df0952f434e9f35bbdf02facdd8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8a969df0952f434e9f35bbdf02facdd8 2024-11-11T22:26:02,932 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8a969df0952f434e9f35bbdf02facdd8, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T22:26:02,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/163e7c53019a4e05a101900d7adabd2f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/163e7c53019a4e05a101900d7adabd2f 2024-11-11T22:26:02,949 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/163e7c53019a4e05a101900d7adabd2f, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T22:26:02,951 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e47b46921ed127eb101b947b07da0745 in 1379ms, sequenceid=118, compaction requested=true 2024-11-11T22:26:02,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:02,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:02,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T22:26:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T22:26:02,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T22:26:02,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4620 sec 2024-11-11T22:26:02,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.4750 sec 2024-11-11T22:26:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:03,024 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:03,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:26:03,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111451c35267ee54a0e9d5cee7bb2371c34_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363961874/Put/seqid=0 2024-11-11T22:26:03,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741985_1161 (size=14644) 2024-11-11T22:26:03,127 INFO [master/8d82b89dd27d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T22:26:03,127 INFO [master/8d82b89dd27d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T22:26:03,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364023164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:03,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364023273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:03,460 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:03,473 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111451c35267ee54a0e9d5cee7bb2371c34_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111451c35267ee54a0e9d5cee7bb2371c34_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:03,475 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8ffea9a3a3334d80bffd94069240ec10, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:03,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8ffea9a3a3334d80bffd94069240ec10 is 175, key is test_row_0/A:col10/1731363961874/Put/seqid=0 2024-11-11T22:26:03,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741986_1162 (size=39599) 2024-11-11T22:26:03,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364023483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:03,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:03,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:03,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:03,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364023793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:03,884 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8ffea9a3a3334d80bffd94069240ec10 2024-11-11T22:26:03,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/c4f86b60970040b5a2dee72fcb52db89 is 50, key is test_row_0/B:col10/1731363961874/Put/seqid=0 2024-11-11T22:26:03,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741987_1163 (size=12051) 2024-11-11T22:26:03,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/c4f86b60970040b5a2dee72fcb52db89 2024-11-11T22:26:03,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/9ce9fdbfe9e44f57986a57486417bc27 is 50, key is test_row_0/C:col10/1731363961874/Put/seqid=0 2024-11-11T22:26:03,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741988_1164 (size=12051) 2024-11-11T22:26:04,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364024303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:04,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/9ce9fdbfe9e44f57986a57486417bc27 2024-11-11T22:26:04,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8ffea9a3a3334d80bffd94069240ec10 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ffea9a3a3334d80bffd94069240ec10 2024-11-11T22:26:04,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ffea9a3a3334d80bffd94069240ec10, entries=200, sequenceid=130, filesize=38.7 K 2024-11-11T22:26:04,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/c4f86b60970040b5a2dee72fcb52db89 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c4f86b60970040b5a2dee72fcb52db89 2024-11-11T22:26:04,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c4f86b60970040b5a2dee72fcb52db89, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T22:26:04,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/9ce9fdbfe9e44f57986a57486417bc27 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9ce9fdbfe9e44f57986a57486417bc27 2024-11-11T22:26:04,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9ce9fdbfe9e44f57986a57486417bc27, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T22:26:04,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e47b46921ed127eb101b947b07da0745 in 1326ms, sequenceid=130, compaction requested=true 2024-11-11T22:26:04,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:04,351 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:04,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:04,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:04,351 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:04,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:04,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:04,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:04,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:04,353 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:04,353 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/B is initiating minor compaction (all files) 2024-11-11T22:26:04,353 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/B in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:04,353 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/e6a6d5f9883d43bebea545323b665c02, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0d2b5c2579654975874ceeb8cae0dde5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8a969df0952f434e9f35bbdf02facdd8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c4f86b60970040b5a2dee72fcb52db89] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=47.1 K 2024-11-11T22:26:04,353 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141196 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:04,354 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/A is initiating minor compaction (all files) 2024-11-11T22:26:04,354 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting e6a6d5f9883d43bebea545323b665c02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363958665 2024-11-11T22:26:04,354 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/A in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:04,354 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/1089999c69fa47ad9ead07e174838ea8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8f48727d119645858732205bac784b9d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/489af0702b55442091a0b58507f38489, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ffea9a3a3334d80bffd94069240ec10] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=137.9 K 2024-11-11T22:26:04,354 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:04,354 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/1089999c69fa47ad9ead07e174838ea8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8f48727d119645858732205bac784b9d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/489af0702b55442091a0b58507f38489, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ffea9a3a3334d80bffd94069240ec10] 2024-11-11T22:26:04,354 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2b5c2579654975874ceeb8cae0dde5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731363959330 2024-11-11T22:26:04,354 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a969df0952f434e9f35bbdf02facdd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731363960677 2024-11-11T22:26:04,354 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1089999c69fa47ad9ead07e174838ea8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363958665 2024-11-11T22:26:04,355 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f86b60970040b5a2dee72fcb52db89, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731363961867 2024-11-11T22:26:04,355 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f48727d119645858732205bac784b9d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731363959330 2024-11-11T22:26:04,356 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 489af0702b55442091a0b58507f38489, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731363960677 2024-11-11T22:26:04,357 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ffea9a3a3334d80bffd94069240ec10, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731363961867 2024-11-11T22:26:04,385 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:04,385 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:04,385 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/68ad88bf8f76419c8c51aa55cbe2795e is 50, key is test_row_0/B:col10/1731363961874/Put/seqid=0 2024-11-11T22:26:04,389 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111fc530bb868f1490ea427ae13e9c42d6c_e47b46921ed127eb101b947b07da0745 store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:04,392 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111fc530bb868f1490ea427ae13e9c42d6c_e47b46921ed127eb101b947b07da0745, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:04,392 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fc530bb868f1490ea427ae13e9c42d6c_e47b46921ed127eb101b947b07da0745 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:04,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741989_1165 (size=12325) 2024-11-11T22:26:04,401 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/68ad88bf8f76419c8c51aa55cbe2795e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/68ad88bf8f76419c8c51aa55cbe2795e 2024-11-11T22:26:04,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741990_1166 (size=4469) 2024-11-11T22:26:04,407 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e47b46921ed127eb101b947b07da0745/B of e47b46921ed127eb101b947b07da0745 into 68ad88bf8f76419c8c51aa55cbe2795e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:04,407 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:04,407 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/B, priority=12, startTime=1731363964351; duration=0sec 2024-11-11T22:26:04,407 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#A#compaction#142 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:04,407 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:04,407 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:B 2024-11-11T22:26:04,407 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:04,408 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/040a5bf9557a46e797fa6646a61980ac is 175, key is test_row_0/A:col10/1731363961874/Put/seqid=0 2024-11-11T22:26:04,411 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:04,411 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/C is initiating minor compaction (all files) 2024-11-11T22:26:04,412 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/C in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:04,412 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4719d4b17565458f94f22649d54843ff, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/54c7faf9001846578658c0367579d019, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/163e7c53019a4e05a101900d7adabd2f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9ce9fdbfe9e44f57986a57486417bc27] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=47.1 K 2024-11-11T22:26:04,413 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4719d4b17565458f94f22649d54843ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363958665 2024-11-11T22:26:04,414 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 54c7faf9001846578658c0367579d019, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731363959330 2024-11-11T22:26:04,414 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 163e7c53019a4e05a101900d7adabd2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731363960677 2024-11-11T22:26:04,415 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ce9fdbfe9e44f57986a57486417bc27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731363961867 2024-11-11T22:26:04,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741991_1167 (size=31279) 2024-11-11T22:26:04,429 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#C#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:04,429 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/73f44159d07f4edf80116c53b0a73c88 is 50, key is test_row_0/C:col10/1731363961874/Put/seqid=0 2024-11-11T22:26:04,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741992_1168 (size=12325) 2024-11-11T22:26:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:26:04,643 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:04,644 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T22:26:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:26:04,645 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:04,646 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:04,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:26:04,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T22:26:04,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:04,799 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:26:04,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111851b4d710cb343f3b1c7087c8d2e588c_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363963157/Put/seqid=0 2024-11-11T22:26:04,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741993_1169 (size=12304) 2024-11-11T22:26:04,830 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/040a5bf9557a46e797fa6646a61980ac as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/040a5bf9557a46e797fa6646a61980ac 2024-11-11T22:26:04,836 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e47b46921ed127eb101b947b07da0745/A of e47b46921ed127eb101b947b07da0745 into 040a5bf9557a46e797fa6646a61980ac(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:04,836 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:04,836 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/A, priority=12, startTime=1731363964351; duration=0sec 2024-11-11T22:26:04,836 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:04,836 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:A 2024-11-11T22:26:04,840 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/73f44159d07f4edf80116c53b0a73c88 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/73f44159d07f4edf80116c53b0a73c88 2024-11-11T22:26:04,846 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e47b46921ed127eb101b947b07da0745/C of e47b46921ed127eb101b947b07da0745 into 73f44159d07f4edf80116c53b0a73c88(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:04,846 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:04,846 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/C, priority=12, startTime=1731363964351; duration=0sec 2024-11-11T22:26:04,846 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:04,846 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:C 2024-11-11T22:26:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:26:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:05,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:05,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364025018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364025022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364025033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364025033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,037 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:05,037 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364025133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364025133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:05,231 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111851b4d710cb343f3b1c7087c8d2e588c_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111851b4d710cb343f3b1c7087c8d2e588c_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:05,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/476c8d23c0d2401698b37ce2ed1d79c4, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:05,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/476c8d23c0d2401698b37ce2ed1d79c4 is 175, key is test_row_0/A:col10/1731363963157/Put/seqid=0 2024-11-11T22:26:05,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741994_1170 (size=31105) 2024-11-11T22:26:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:26:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364025313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,315 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364025343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364025343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,641 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/476c8d23c0d2401698b37ce2ed1d79c4 2024-11-11T22:26:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/84ec6db24bb24a4281f7807b0bbd7375 is 50, key is test_row_0/B:col10/1731363963157/Put/seqid=0 2024-11-11T22:26:05,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741995_1171 (size=12151) 2024-11-11T22:26:05,654 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/84ec6db24bb24a4281f7807b0bbd7375 2024-11-11T22:26:05,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364025654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364025653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/89aba986189f4cfe927290c87aaf3879 is 50, key is test_row_0/C:col10/1731363963157/Put/seqid=0 2024-11-11T22:26:05,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741996_1172 (size=12151) 2024-11-11T22:26:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:26:06,069 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/89aba986189f4cfe927290c87aaf3879 2024-11-11T22:26:06,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/476c8d23c0d2401698b37ce2ed1d79c4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/476c8d23c0d2401698b37ce2ed1d79c4 2024-11-11T22:26:06,077 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/476c8d23c0d2401698b37ce2ed1d79c4, entries=150, sequenceid=155, filesize=30.4 K 2024-11-11T22:26:06,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/84ec6db24bb24a4281f7807b0bbd7375 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/84ec6db24bb24a4281f7807b0bbd7375 2024-11-11T22:26:06,082 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/84ec6db24bb24a4281f7807b0bbd7375, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T22:26:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/89aba986189f4cfe927290c87aaf3879 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/89aba986189f4cfe927290c87aaf3879 2024-11-11T22:26:06,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/89aba986189f4cfe927290c87aaf3879, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T22:26:06,088 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e47b46921ed127eb101b947b07da0745 in 1289ms, sequenceid=155, compaction requested=false 2024-11-11T22:26:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T22:26:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T22:26:06,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T22:26:06,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4440 sec 2024-11-11T22:26:06,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4480 sec 2024-11-11T22:26:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:06,164 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:06,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:26:06,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110da1920e204240048a239a6731f6d65f_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363966163/Put/seqid=0 2024-11-11T22:26:06,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741997_1173 (size=17284) 2024-11-11T22:26:06,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364026200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:06,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:06,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364026204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:06,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:06,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364026313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:06,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364026314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:06,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364026523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:06,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364026523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:06,582 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:06,587 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110da1920e204240048a239a6731f6d65f_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110da1920e204240048a239a6731f6d65f_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:06,588 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/e7f4f11934c94670868891784b661f1d, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:06,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/e7f4f11934c94670868891784b661f1d is 175, key is test_row_0/A:col10/1731363966163/Put/seqid=0 2024-11-11T22:26:06,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741998_1174 (size=48389) 2024-11-11T22:26:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:26:06,803 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:06,803 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T22:26:06,805 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:26:06,806 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:06,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:06,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364026833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:06,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364026833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:26:06,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:26:06,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:06,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:06,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:06,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:06,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:06,993 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/e7f4f11934c94670868891784b661f1d 2024-11-11T22:26:07,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/c57b9fb7233a44178d600a00a6b69a33 is 50, key is test_row_0/B:col10/1731363966163/Put/seqid=0 2024-11-11T22:26:07,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741999_1175 (size=12151) 2024-11-11T22:26:07,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:26:07,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:07,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:26:07,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:26:07,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:07,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364027343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:07,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364027343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:07,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364027343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:07,345 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:07,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/c57b9fb7233a44178d600a00a6b69a33 2024-11-11T22:26:07,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:26:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/4187fb40962f40cf9cdc29409faa159e is 50, key is test_row_0/C:col10/1731363966163/Put/seqid=0 2024-11-11T22:26:07,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742000_1176 (size=12151) 2024-11-11T22:26:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:26:07,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:26:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:26:07,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:07,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:07,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/4187fb40962f40cf9cdc29409faa159e 2024-11-11T22:26:07,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/e7f4f11934c94670868891784b661f1d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e7f4f11934c94670868891784b661f1d 2024-11-11T22:26:07,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e7f4f11934c94670868891784b661f1d, entries=250, sequenceid=171, filesize=47.3 K 2024-11-11T22:26:07,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/c57b9fb7233a44178d600a00a6b69a33 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c57b9fb7233a44178d600a00a6b69a33 2024-11-11T22:26:07,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c57b9fb7233a44178d600a00a6b69a33, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T22:26:07,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/4187fb40962f40cf9cdc29409faa159e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4187fb40962f40cf9cdc29409faa159e 2024-11-11T22:26:07,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4187fb40962f40cf9cdc29409faa159e, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T22:26:07,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e47b46921ed127eb101b947b07da0745 in 1688ms, sequenceid=171, compaction requested=true 2024-11-11T22:26:07,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:07,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:07,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:07,852 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:07,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:07,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:07,853 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:07,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:07,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:07,855 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:07,855 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/A is initiating minor compaction (all files) 2024-11-11T22:26:07,855 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/A in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,856 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/040a5bf9557a46e797fa6646a61980ac, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/476c8d23c0d2401698b37ce2ed1d79c4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e7f4f11934c94670868891784b661f1d] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=108.2 K 2024-11-11T22:26:07,856 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,856 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/040a5bf9557a46e797fa6646a61980ac, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/476c8d23c0d2401698b37ce2ed1d79c4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e7f4f11934c94670868891784b661f1d] 2024-11-11T22:26:07,856 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:07,856 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 040a5bf9557a46e797fa6646a61980ac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731363961867 2024-11-11T22:26:07,856 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/B is initiating minor compaction (all files) 2024-11-11T22:26:07,856 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/B in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,857 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/68ad88bf8f76419c8c51aa55cbe2795e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/84ec6db24bb24a4281f7807b0bbd7375, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c57b9fb7233a44178d600a00a6b69a33] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=35.8 K 2024-11-11T22:26:07,857 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 476c8d23c0d2401698b37ce2ed1d79c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731363963155 2024-11-11T22:26:07,857 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7f4f11934c94670868891784b661f1d, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731363965015 2024-11-11T22:26:07,857 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 68ad88bf8f76419c8c51aa55cbe2795e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731363961867 2024-11-11T22:26:07,857 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 84ec6db24bb24a4281f7807b0bbd7375, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731363963155 2024-11-11T22:26:07,858 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting c57b9fb7233a44178d600a00a6b69a33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731363965017 2024-11-11T22:26:07,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:26:07,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,877 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T22:26:07,883 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:07,884 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/a1586540c4734b1b9227b860c8d4c9da is 50, key is test_row_0/B:col10/1731363966163/Put/seqid=0 2024-11-11T22:26:07,886 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a7326106fa22413fb3f12e652618b86b_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363966198/Put/seqid=0 2024-11-11T22:26:07,921 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411115e1b7a39c74e4657ae00ee8dd2932030_e47b46921ed127eb101b947b07da0745 store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:07,924 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411115e1b7a39c74e4657ae00ee8dd2932030_e47b46921ed127eb101b947b07da0745, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:07,924 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115e1b7a39c74e4657ae00ee8dd2932030_e47b46921ed127eb101b947b07da0745 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:07,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742001_1177 (size=12527) 2024-11-11T22:26:07,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:26:07,945 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/a1586540c4734b1b9227b860c8d4c9da as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/a1586540c4734b1b9227b860c8d4c9da 2024-11-11T22:26:07,957 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e47b46921ed127eb101b947b07da0745/B of e47b46921ed127eb101b947b07da0745 into a1586540c4734b1b9227b860c8d4c9da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:07,957 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:07,957 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/B, priority=13, startTime=1731363967852; duration=0sec 2024-11-11T22:26:07,958 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:07,958 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:B 2024-11-11T22:26:07,958 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:07,962 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:07,962 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/C is initiating minor compaction (all files) 2024-11-11T22:26:07,962 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/C in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:07,962 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/73f44159d07f4edf80116c53b0a73c88, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/89aba986189f4cfe927290c87aaf3879, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4187fb40962f40cf9cdc29409faa159e] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=35.8 K 2024-11-11T22:26:07,963 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 73f44159d07f4edf80116c53b0a73c88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731363961867 2024-11-11T22:26:07,963 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 89aba986189f4cfe927290c87aaf3879, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731363963155 2024-11-11T22:26:07,964 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4187fb40962f40cf9cdc29409faa159e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731363965017 2024-11-11T22:26:07,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742003_1179 (size=4469) 2024-11-11T22:26:07,987 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#A#compaction#151 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:07,988 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/d26bb4adc9c84153a3bf40c1227ad87f is 175, key is test_row_0/A:col10/1731363966163/Put/seqid=0 2024-11-11T22:26:07,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742002_1178 (size=12304) 2024-11-11T22:26:08,015 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:08,016 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/67f6d9beb1494e6eb2a5b9f20277033b is 50, key is test_row_0/C:col10/1731363966163/Put/seqid=0 2024-11-11T22:26:08,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742004_1180 (size=31481) 2024-11-11T22:26:08,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742005_1181 (size=12527) 2024-11-11T22:26:08,069 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/67f6d9beb1494e6eb2a5b9f20277033b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/67f6d9beb1494e6eb2a5b9f20277033b 2024-11-11T22:26:08,083 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e47b46921ed127eb101b947b07da0745/C of e47b46921ed127eb101b947b07da0745 into 67f6d9beb1494e6eb2a5b9f20277033b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:08,084 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:08,084 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/C, priority=13, startTime=1731363967853; duration=0sec 2024-11-11T22:26:08,084 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:08,084 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:C 2024-11-11T22:26:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:08,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:08,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364028393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:08,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:08,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364028398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:08,411 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a7326106fa22413fb3f12e652618b86b_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a7326106fa22413fb3f12e652618b86b_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/26f5783ed2b7447f8e8462592257097e, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:08,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/26f5783ed2b7447f8e8462592257097e is 175, key is test_row_0/A:col10/1731363966198/Put/seqid=0 2024-11-11T22:26:08,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742006_1182 (size=31105) 2024-11-11T22:26:08,433 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/26f5783ed2b7447f8e8462592257097e 2024-11-11T22:26:08,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/8449b8b63b77493dad5bef6f740aa8ac is 50, key is test_row_0/B:col10/1731363966198/Put/seqid=0 2024-11-11T22:26:08,460 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/d26bb4adc9c84153a3bf40c1227ad87f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/d26bb4adc9c84153a3bf40c1227ad87f 2024-11-11T22:26:08,469 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e47b46921ed127eb101b947b07da0745/A of e47b46921ed127eb101b947b07da0745 into d26bb4adc9c84153a3bf40c1227ad87f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:08,470 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:08,470 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/A, priority=13, startTime=1731363967852; duration=0sec 2024-11-11T22:26:08,470 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:08,470 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:A 2024-11-11T22:26:08,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742007_1183 (size=12151) 2024-11-11T22:26:08,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364028503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:08,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364028513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:08,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364028713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:08,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364028723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:08,872 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/8449b8b63b77493dad5bef6f740aa8ac 2024-11-11T22:26:08,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/665c6a204d934821a9210caadfe7e235 is 50, key is test_row_0/C:col10/1731363966198/Put/seqid=0 2024-11-11T22:26:08,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742008_1184 (size=12151) 2024-11-11T22:26:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:26:09,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364029026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:09,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364029034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:09,296 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/665c6a204d934821a9210caadfe7e235 2024-11-11T22:26:09,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/26f5783ed2b7447f8e8462592257097e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/26f5783ed2b7447f8e8462592257097e 2024-11-11T22:26:09,310 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/26f5783ed2b7447f8e8462592257097e, entries=150, sequenceid=193, filesize=30.4 K 2024-11-11T22:26:09,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/8449b8b63b77493dad5bef6f740aa8ac as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8449b8b63b77493dad5bef6f740aa8ac 2024-11-11T22:26:09,317 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8449b8b63b77493dad5bef6f740aa8ac, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T22:26:09,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/665c6a204d934821a9210caadfe7e235 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/665c6a204d934821a9210caadfe7e235 2024-11-11T22:26:09,340 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/665c6a204d934821a9210caadfe7e235, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T22:26:09,341 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e47b46921ed127eb101b947b07da0745 in 1464ms, sequenceid=193, compaction requested=false 2024-11-11T22:26:09,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:09,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:09,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T22:26:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T22:26:09,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T22:26:09,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5370 sec 2024-11-11T22:26:09,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.5430 sec 2024-11-11T22:26:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:09,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:09,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T22:26:09,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111136c0c35072ce434db7e9338553476fdb_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363969533/Put/seqid=0 2024-11-11T22:26:09,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742009_1185 (size=14794) 2024-11-11T22:26:09,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:09,611 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111136c0c35072ce434db7e9338553476fdb_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111136c0c35072ce434db7e9338553476fdb_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:09,616 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/914509cc3ca9453f8202cb41e2d80c4c, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:09,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/914509cc3ca9453f8202cb41e2d80c4c is 175, key is test_row_0/A:col10/1731363969533/Put/seqid=0 2024-11-11T22:26:09,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364029617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:09,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364029627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:09,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742010_1186 (size=39749) 2024-11-11T22:26:09,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364029734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364029734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:09,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364029943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364029943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:10,042 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/914509cc3ca9453f8202cb41e2d80c4c 2024-11-11T22:26:10,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/b630bc44187a447dbea8ebab98656f2e is 50, key is test_row_0/B:col10/1731363969533/Put/seqid=0 2024-11-11T22:26:10,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742011_1187 (size=12151) 2024-11-11T22:26:10,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/b630bc44187a447dbea8ebab98656f2e 2024-11-11T22:26:10,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/40a28dda743244db8d4256cc65a81139 is 50, key is test_row_0/C:col10/1731363969533/Put/seqid=0 2024-11-11T22:26:10,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742012_1188 (size=12151) 2024-11-11T22:26:10,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364030253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:10,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364030253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:10,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/40a28dda743244db8d4256cc65a81139 2024-11-11T22:26:10,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/914509cc3ca9453f8202cb41e2d80c4c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/914509cc3ca9453f8202cb41e2d80c4c 2024-11-11T22:26:10,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/914509cc3ca9453f8202cb41e2d80c4c, entries=200, sequenceid=211, filesize=38.8 K 2024-11-11T22:26:10,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/b630bc44187a447dbea8ebab98656f2e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/b630bc44187a447dbea8ebab98656f2e 2024-11-11T22:26:10,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/b630bc44187a447dbea8ebab98656f2e, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T22:26:10,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/40a28dda743244db8d4256cc65a81139 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/40a28dda743244db8d4256cc65a81139 2024-11-11T22:26:10,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/40a28dda743244db8d4256cc65a81139, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T22:26:10,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e47b46921ed127eb101b947b07da0745 in 1007ms, sequenceid=211, compaction requested=true 2024-11-11T22:26:10,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:10,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:10,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:10,542 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:10,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:10,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:10,542 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:10,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:10,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:10,545 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:10,545 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/B is initiating minor compaction (all files) 2024-11-11T22:26:10,545 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/B in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:10,545 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/a1586540c4734b1b9227b860c8d4c9da, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8449b8b63b77493dad5bef6f740aa8ac, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/b630bc44187a447dbea8ebab98656f2e] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=36.0 K 2024-11-11T22:26:10,545 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a1586540c4734b1b9227b860c8d4c9da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731363965017 2024-11-11T22:26:10,546 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:10,546 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/A is initiating minor compaction (all files) 2024-11-11T22:26:10,546 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/A in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:10,546 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/d26bb4adc9c84153a3bf40c1227ad87f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/26f5783ed2b7447f8e8462592257097e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/914509cc3ca9453f8202cb41e2d80c4c] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=99.9 K 2024-11-11T22:26:10,546 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:10,546 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/d26bb4adc9c84153a3bf40c1227ad87f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/26f5783ed2b7447f8e8462592257097e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/914509cc3ca9453f8202cb41e2d80c4c] 2024-11-11T22:26:10,547 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 8449b8b63b77493dad5bef6f740aa8ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731363966184 2024-11-11T22:26:10,547 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting d26bb4adc9c84153a3bf40c1227ad87f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731363965017 2024-11-11T22:26:10,547 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting b630bc44187a447dbea8ebab98656f2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731363968361 2024-11-11T22:26:10,548 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26f5783ed2b7447f8e8462592257097e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731363966184 2024-11-11T22:26:10,548 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 914509cc3ca9453f8202cb41e2d80c4c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731363968361 2024-11-11T22:26:10,579 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:10,579 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/74459a62b2ef46f89268c747816de850 is 50, key is test_row_0/B:col10/1731363969533/Put/seqid=0 2024-11-11T22:26:10,589 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:10,607 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111af46e5d967394d22aa10239cbb450694_e47b46921ed127eb101b947b07da0745 store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:10,610 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111af46e5d967394d22aa10239cbb450694_e47b46921ed127eb101b947b07da0745, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:10,610 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111af46e5d967394d22aa10239cbb450694_e47b46921ed127eb101b947b07da0745 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:10,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742013_1189 (size=12629) 2024-11-11T22:26:10,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742014_1190 (size=4469) 2024-11-11T22:26:10,662 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#A#compaction#160 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:10,663 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/426ea16c171b4d3e8f92decd6451734f is 175, key is test_row_0/A:col10/1731363969533/Put/seqid=0 2024-11-11T22:26:10,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742015_1191 (size=31583) 2024-11-11T22:26:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:10,764 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:10,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T22:26:10,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116ea49f8f22994952b463eb5962d017d1_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363970763/Put/seqid=0 2024-11-11T22:26:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364030814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364030815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:10,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742016_1192 (size=12304) 2024-11-11T22:26:10,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364030925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:10,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364030928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:26:10,984 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:10,984 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T22:26:10,986 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:26:10,987 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:10,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:11,043 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/74459a62b2ef46f89268c747816de850 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/74459a62b2ef46f89268c747816de850 2024-11-11T22:26:11,063 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e47b46921ed127eb101b947b07da0745/B of e47b46921ed127eb101b947b07da0745 into 74459a62b2ef46f89268c747816de850(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:11,063 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:11,063 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/B, priority=13, startTime=1731363970542; duration=0sec 2024-11-11T22:26:11,064 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:11,064 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:B 2024-11-11T22:26:11,064 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:11,075 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:11,075 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/C is initiating minor compaction (all files) 2024-11-11T22:26:11,075 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/C in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,075 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/67f6d9beb1494e6eb2a5b9f20277033b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/665c6a204d934821a9210caadfe7e235, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/40a28dda743244db8d4256cc65a81139] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=36.0 K 2024-11-11T22:26:11,075 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 67f6d9beb1494e6eb2a5b9f20277033b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731363965017 2024-11-11T22:26:11,076 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 665c6a204d934821a9210caadfe7e235, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731363966184 2024-11-11T22:26:11,077 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a28dda743244db8d4256cc65a81139, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731363968361 2024-11-11T22:26:11,079 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/426ea16c171b4d3e8f92decd6451734f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/426ea16c171b4d3e8f92decd6451734f 2024-11-11T22:26:11,087 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:11,087 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/de5aeac813424d6db51475e3217a83b3 is 50, key is test_row_0/C:col10/1731363969533/Put/seqid=0 2024-11-11T22:26:11,087 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e47b46921ed127eb101b947b07da0745/A of e47b46921ed127eb101b947b07da0745 into 426ea16c171b4d3e8f92decd6451734f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:11,087 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:11,087 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/A, priority=13, startTime=1731363970542; duration=0sec 2024-11-11T22:26:11,087 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:11,087 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:A 2024-11-11T22:26:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:26:11,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742017_1193 (size=12629) 2024-11-11T22:26:11,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364031134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:11,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364031134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:11,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T22:26:11,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:11,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,246 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:11,251 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116ea49f8f22994952b463eb5962d017d1_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116ea49f8f22994952b463eb5962d017d1_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:11,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/628cd9433af647a18d5018ec844eeca5, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:11,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/628cd9433af647a18d5018ec844eeca5 is 175, key is test_row_0/A:col10/1731363970763/Put/seqid=0 2024-11-11T22:26:11,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742018_1194 (size=31105) 2024-11-11T22:26:11,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T22:26:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:26:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35684 deadline: 1731364031383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:11,390 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:11,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364031443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:11,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364031443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:11,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T22:26:11,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:11,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:11,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,526 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/de5aeac813424d6db51475e3217a83b3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/de5aeac813424d6db51475e3217a83b3 2024-11-11T22:26:11,533 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e47b46921ed127eb101b947b07da0745/C of e47b46921ed127eb101b947b07da0745 into de5aeac813424d6db51475e3217a83b3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:11,533 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:11,533 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/C, priority=13, startTime=1731363970542; duration=0sec 2024-11-11T22:26:11,533 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:11,533 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:C 2024-11-11T22:26:11,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T22:26:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:26:11,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/628cd9433af647a18d5018ec844eeca5 2024-11-11T22:26:11,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/5350eea2c0524036b06c7d2589e5124d is 50, key is test_row_0/B:col10/1731363970763/Put/seqid=0 2024-11-11T22:26:11,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T22:26:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742019_1195 (size=12151) 2024-11-11T22:26:11,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T22:26:11,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:11,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:11,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364031953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364031953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:12,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T22:26:12,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:12,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:12,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:26:12,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/5350eea2c0524036b06c7d2589e5124d 2024-11-11T22:26:12,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/ef475e61a41c4b7fb8b51169ba2b6fd7 is 50, key is test_row_0/C:col10/1731363970763/Put/seqid=0 2024-11-11T22:26:12,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T22:26:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:12,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:12,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742020_1196 (size=12151) 2024-11-11T22:26:12,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/ef475e61a41c4b7fb8b51169ba2b6fd7 2024-11-11T22:26:12,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/628cd9433af647a18d5018ec844eeca5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/628cd9433af647a18d5018ec844eeca5 2024-11-11T22:26:12,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/628cd9433af647a18d5018ec844eeca5, entries=150, sequenceid=233, filesize=30.4 K 2024-11-11T22:26:12,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/5350eea2c0524036b06c7d2589e5124d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/5350eea2c0524036b06c7d2589e5124d 2024-11-11T22:26:12,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/5350eea2c0524036b06c7d2589e5124d, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T22:26:12,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/ef475e61a41c4b7fb8b51169ba2b6fd7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/ef475e61a41c4b7fb8b51169ba2b6fd7 2024-11-11T22:26:12,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/ef475e61a41c4b7fb8b51169ba2b6fd7, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T22:26:12,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e47b46921ed127eb101b947b07da0745 in 1518ms, sequenceid=233, compaction requested=false 2024-11-11T22:26:12,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:12,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T22:26:12,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:12,379 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:26:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111afb896c3c0474822b69dded484b27f46_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363970813/Put/seqid=0 2024-11-11T22:26:12,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742021_1197 (size=12304) 2024-11-11T22:26:12,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:12,820 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111afb896c3c0474822b69dded484b27f46_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111afb896c3c0474822b69dded484b27f46_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:12,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/4d28f677793e4f1e94613053772c7756, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:12,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/4d28f677793e4f1e94613053772c7756 is 175, key is test_row_0/A:col10/1731363970813/Put/seqid=0 2024-11-11T22:26:12,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742022_1198 (size=31105) 2024-11-11T22:26:12,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:13,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364033113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364033113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:26:13,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364033223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364033223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:13,276 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/4d28f677793e4f1e94613053772c7756 2024-11-11T22:26:13,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/6983cd0d3b344c7394fb6efec54acc72 is 50, key is test_row_0/B:col10/1731363970813/Put/seqid=0 2024-11-11T22:26:13,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742023_1199 (size=12151) 2024-11-11T22:26:13,296 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/6983cd0d3b344c7394fb6efec54acc72 2024-11-11T22:26:13,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/124ca945df2b4d92b9065b34e05713e2 is 50, key is test_row_0/C:col10/1731363970813/Put/seqid=0 2024-11-11T22:26:13,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742024_1200 (size=12151) 2024-11-11T22:26:13,354 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/124ca945df2b4d92b9065b34e05713e2 2024-11-11T22:26:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/4d28f677793e4f1e94613053772c7756 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/4d28f677793e4f1e94613053772c7756 2024-11-11T22:26:13,366 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/4d28f677793e4f1e94613053772c7756, entries=150, sequenceid=250, filesize=30.4 K 2024-11-11T22:26:13,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/6983cd0d3b344c7394fb6efec54acc72 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/6983cd0d3b344c7394fb6efec54acc72 2024-11-11T22:26:13,374 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/6983cd0d3b344c7394fb6efec54acc72, entries=150, sequenceid=250, filesize=11.9 K 2024-11-11T22:26:13,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/124ca945df2b4d92b9065b34e05713e2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/124ca945df2b4d92b9065b34e05713e2 2024-11-11T22:26:13,381 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/124ca945df2b4d92b9065b34e05713e2, entries=150, sequenceid=250, filesize=11.9 K 2024-11-11T22:26:13,382 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for e47b46921ed127eb101b947b07da0745 in 1003ms, sequenceid=250, compaction requested=true 2024-11-11T22:26:13,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:13,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:13,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T22:26:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T22:26:13,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T22:26:13,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3980 sec 2024-11-11T22:26:13,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.4050 sec 2024-11-11T22:26:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:13,439 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:13,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:26:13,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ca41fb97c9bd4851b0c5e44b634f7f00_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363973111/Put/seqid=0 2024-11-11T22:26:13,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364033486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:13,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364033489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:13,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742025_1201 (size=14994) 2024-11-11T22:26:13,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:13,521 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ca41fb97c9bd4851b0c5e44b634f7f00_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ca41fb97c9bd4851b0c5e44b634f7f00_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:13,522 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8ad89e94ac1e4f25a01def8be3af0d18, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:13,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8ad89e94ac1e4f25a01def8be3af0d18 is 175, key is test_row_0/A:col10/1731363973111/Put/seqid=0 2024-11-11T22:26:13,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742026_1202 (size=39949) 2024-11-11T22:26:13,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8ad89e94ac1e4f25a01def8be3af0d18 2024-11-11T22:26:13,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/57db733ca2124e41a8e685c1c528def2 is 50, key is test_row_0/B:col10/1731363973111/Put/seqid=0 2024-11-11T22:26:13,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364033603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:13,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364033603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:13,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742027_1203 (size=12301) 2024-11-11T22:26:13,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364033813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:13,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364033813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:14,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/57db733ca2124e41a8e685c1c528def2 2024-11-11T22:26:14,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/b723008ad14b46e3b00d15cc747660a9 is 50, key is test_row_0/C:col10/1731363973111/Put/seqid=0 2024-11-11T22:26:14,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742028_1204 (size=12301) 2024-11-11T22:26:14,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/b723008ad14b46e3b00d15cc747660a9 2024-11-11T22:26:14,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8ad89e94ac1e4f25a01def8be3af0d18 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ad89e94ac1e4f25a01def8be3af0d18 2024-11-11T22:26:14,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ad89e94ac1e4f25a01def8be3af0d18, entries=200, sequenceid=274, filesize=39.0 K 2024-11-11T22:26:14,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/57db733ca2124e41a8e685c1c528def2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/57db733ca2124e41a8e685c1c528def2 2024-11-11T22:26:14,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/57db733ca2124e41a8e685c1c528def2, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T22:26:14,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/b723008ad14b46e3b00d15cc747660a9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/b723008ad14b46e3b00d15cc747660a9 2024-11-11T22:26:14,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/b723008ad14b46e3b00d15cc747660a9, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T22:26:14,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e47b46921ed127eb101b947b07da0745 in 660ms, sequenceid=274, compaction requested=true 2024-11-11T22:26:14,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:14,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:14,099 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:14,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:14,099 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:14,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:14,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:14,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:14,101 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:14,101 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:14,101 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/A is initiating minor compaction (all files) 2024-11-11T22:26:14,101 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/B is initiating minor compaction (all files) 2024-11-11T22:26:14,101 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/B in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:14,101 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/A in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:14,101 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/426ea16c171b4d3e8f92decd6451734f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/628cd9433af647a18d5018ec844eeca5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/4d28f677793e4f1e94613053772c7756, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ad89e94ac1e4f25a01def8be3af0d18] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=130.6 K 2024-11-11T22:26:14,101 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/74459a62b2ef46f89268c747816de850, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/5350eea2c0524036b06c7d2589e5124d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/6983cd0d3b344c7394fb6efec54acc72, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/57db733ca2124e41a8e685c1c528def2] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=48.1 K 2024-11-11T22:26:14,101 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:14,101 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/426ea16c171b4d3e8f92decd6451734f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/628cd9433af647a18d5018ec844eeca5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/4d28f677793e4f1e94613053772c7756, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ad89e94ac1e4f25a01def8be3af0d18] 2024-11-11T22:26:14,102 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 426ea16c171b4d3e8f92decd6451734f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731363968361 2024-11-11T22:26:14,102 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 74459a62b2ef46f89268c747816de850, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731363968361 2024-11-11T22:26:14,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:14,102 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 5350eea2c0524036b06c7d2589e5124d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731363969586 2024-11-11T22:26:14,103 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 628cd9433af647a18d5018ec844eeca5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731363969586 2024-11-11T22:26:14,103 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 6983cd0d3b344c7394fb6efec54acc72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731363970791 2024-11-11T22:26:14,103 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d28f677793e4f1e94613053772c7756, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731363970791 2024-11-11T22:26:14,103 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 57db733ca2124e41a8e685c1c528def2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731363973082 2024-11-11T22:26:14,105 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad89e94ac1e4f25a01def8be3af0d18, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731363973082 2024-11-11T22:26:14,119 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:14,127 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111bbe3cc88044b496cbb09bd916faf22d9_e47b46921ed127eb101b947b07da0745 store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:14,127 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#B#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:14,127 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/0cf48566ad6f475991d3a37a2715a8a3 is 50, key is test_row_0/B:col10/1731363973111/Put/seqid=0 2024-11-11T22:26:14,129 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111bbe3cc88044b496cbb09bd916faf22d9_e47b46921ed127eb101b947b07da0745, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:14,130 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bbe3cc88044b496cbb09bd916faf22d9_e47b46921ed127eb101b947b07da0745 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:14,131 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:14,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:26:14,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742029_1205 (size=12915) 2024-11-11T22:26:14,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c9486fb48e2a4ccbad033937c481dbb0_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363974127/Put/seqid=0 2024-11-11T22:26:14,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742030_1206 (size=4469) 2024-11-11T22:26:14,165 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#A#compaction#171 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:14,166 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/27f40aaddd31483b80a5e0afdcfdfda0 is 175, key is test_row_0/A:col10/1731363973111/Put/seqid=0 2024-11-11T22:26:14,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742031_1207 (size=12454) 2024-11-11T22:26:14,195 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:14,200 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c9486fb48e2a4ccbad033937c481dbb0_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c9486fb48e2a4ccbad033937c481dbb0_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:14,202 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8393a90852c54c72ae3ee579d68d157d, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:14,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8393a90852c54c72ae3ee579d68d157d is 175, key is test_row_0/A:col10/1731363974127/Put/seqid=0 2024-11-11T22:26:14,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742032_1208 (size=31869) 2024-11-11T22:26:14,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364034214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:14,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:14,229 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/27f40aaddd31483b80a5e0afdcfdfda0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/27f40aaddd31483b80a5e0afdcfdfda0 2024-11-11T22:26:14,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364034223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:14,235 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e47b46921ed127eb101b947b07da0745/A of e47b46921ed127eb101b947b07da0745 into 27f40aaddd31483b80a5e0afdcfdfda0(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:14,235 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:14,235 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/A, priority=12, startTime=1731363974099; duration=0sec 2024-11-11T22:26:14,235 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:14,235 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:A 2024-11-11T22:26:14,235 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:14,238 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:14,238 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/C is initiating minor compaction (all files) 2024-11-11T22:26:14,238 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/C in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:14,238 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/de5aeac813424d6db51475e3217a83b3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/ef475e61a41c4b7fb8b51169ba2b6fd7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/124ca945df2b4d92b9065b34e05713e2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/b723008ad14b46e3b00d15cc747660a9] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=48.1 K 2024-11-11T22:26:14,238 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting de5aeac813424d6db51475e3217a83b3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731363968361 2024-11-11T22:26:14,239 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef475e61a41c4b7fb8b51169ba2b6fd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731363969586 2024-11-11T22:26:14,239 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 124ca945df2b4d92b9065b34e05713e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731363970791 2024-11-11T22:26:14,239 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b723008ad14b46e3b00d15cc747660a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731363973082 2024-11-11T22:26:14,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742033_1209 (size=31255) 2024-11-11T22:26:14,265 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:14,265 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/282c88f7f3a841f0beaff90a909d78fa is 50, key is test_row_0/C:col10/1731363973111/Put/seqid=0 2024-11-11T22:26:14,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742034_1210 (size=12915) 2024-11-11T22:26:14,297 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/282c88f7f3a841f0beaff90a909d78fa as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/282c88f7f3a841f0beaff90a909d78fa 2024-11-11T22:26:14,304 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e47b46921ed127eb101b947b07da0745/C of e47b46921ed127eb101b947b07da0745 into 282c88f7f3a841f0beaff90a909d78fa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:14,304 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:14,304 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/C, priority=12, startTime=1731363974100; duration=0sec 2024-11-11T22:26:14,304 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:14,304 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:C 2024-11-11T22:26:14,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364034334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364034343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:14,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364034544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:14,552 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/0cf48566ad6f475991d3a37a2715a8a3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0cf48566ad6f475991d3a37a2715a8a3 2024-11-11T22:26:14,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364034554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:14,563 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e47b46921ed127eb101b947b07da0745/B of e47b46921ed127eb101b947b07da0745 into 0cf48566ad6f475991d3a37a2715a8a3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:14,563 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:14,563 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/B, priority=12, startTime=1731363974099; duration=0sec 2024-11-11T22:26:14,564 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:14,564 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:B 2024-11-11T22:26:14,659 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8393a90852c54c72ae3ee579d68d157d 2024-11-11T22:26:14,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/06fcf5b1047046debe33e91ecba7c6c0 is 50, key is test_row_0/B:col10/1731363974127/Put/seqid=0 2024-11-11T22:26:14,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742035_1211 (size=12301) 2024-11-11T22:26:14,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/06fcf5b1047046debe33e91ecba7c6c0 2024-11-11T22:26:14,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/7f88d3b7be9d412f97a42bd52d5200f0 is 50, key is test_row_0/C:col10/1731363974127/Put/seqid=0 2024-11-11T22:26:14,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742036_1212 (size=12301) 2024-11-11T22:26:14,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/7f88d3b7be9d412f97a42bd52d5200f0 2024-11-11T22:26:14,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/8393a90852c54c72ae3ee579d68d157d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8393a90852c54c72ae3ee579d68d157d 2024-11-11T22:26:14,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8393a90852c54c72ae3ee579d68d157d, entries=150, sequenceid=288, filesize=30.5 K 2024-11-11T22:26:14,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/06fcf5b1047046debe33e91ecba7c6c0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/06fcf5b1047046debe33e91ecba7c6c0 2024-11-11T22:26:14,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/06fcf5b1047046debe33e91ecba7c6c0, entries=150, sequenceid=288, filesize=12.0 K 2024-11-11T22:26:14,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/7f88d3b7be9d412f97a42bd52d5200f0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/7f88d3b7be9d412f97a42bd52d5200f0 2024-11-11T22:26:14,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/7f88d3b7be9d412f97a42bd52d5200f0, entries=150, sequenceid=288, filesize=12.0 K 2024-11-11T22:26:14,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e47b46921ed127eb101b947b07da0745 in 689ms, sequenceid=288, compaction requested=false 2024-11-11T22:26:14,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:14,857 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:14,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:26:14,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b8672a8180d3492a8077b0a0386e2738_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363974856/Put/seqid=0 2024-11-11T22:26:14,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364034913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:14,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364034920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:14,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742037_1213 (size=14994) 2024-11-11T22:26:14,935 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:14,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b8672a8180d3492a8077b0a0386e2738_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b8672a8180d3492a8077b0a0386e2738_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:14,942 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/b577bb2bfea34778a340d6a9be05e9b8, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:14,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/b577bb2bfea34778a340d6a9be05e9b8 is 175, key is test_row_0/A:col10/1731363974856/Put/seqid=0 2024-11-11T22:26:14,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742038_1214 (size=39949) 2024-11-11T22:26:14,954 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/b577bb2bfea34778a340d6a9be05e9b8 2024-11-11T22:26:14,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/d8daa3dbdb684d7196aa50e78e5f2334 is 50, key is test_row_0/B:col10/1731363974856/Put/seqid=0 2024-11-11T22:26:15,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742039_1215 (size=12301) 2024-11-11T22:26:15,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/d8daa3dbdb684d7196aa50e78e5f2334 2024-11-11T22:26:15,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364035023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/18a2ba0cf479413cb5552f094efd2e32 is 50, key is test_row_0/C:col10/1731363974856/Put/seqid=0 2024-11-11T22:26:15,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364035033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35700 deadline: 1731364035074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,077 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18285 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:15,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742040_1216 (size=12301) 2024-11-11T22:26:15,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/18a2ba0cf479413cb5552f094efd2e32 2024-11-11T22:26:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35712 deadline: 1731364035093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/b577bb2bfea34778a340d6a9be05e9b8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/b577bb2bfea34778a340d6a9be05e9b8 2024-11-11T22:26:15,095 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in e47b46921ed127eb101b947b07da0745 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18302 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/b577bb2bfea34778a340d6a9be05e9b8, entries=200, sequenceid=314, filesize=39.0 K 2024-11-11T22:26:15,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/d8daa3dbdb684d7196aa50e78e5f2334 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d8daa3dbdb684d7196aa50e78e5f2334 2024-11-11T22:26:15,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d8daa3dbdb684d7196aa50e78e5f2334, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T22:26:15,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/18a2ba0cf479413cb5552f094efd2e32 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/18a2ba0cf479413cb5552f094efd2e32 2024-11-11T22:26:15,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/18a2ba0cf479413cb5552f094efd2e32, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T22:26:15,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e47b46921ed127eb101b947b07da0745 in 273ms, sequenceid=314, compaction requested=true 2024-11-11T22:26:15,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:15,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:15,130 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:15,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:15,131 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:15,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:15,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:15,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e47b46921ed127eb101b947b07da0745:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:15,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:15,132 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:15,132 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/A is initiating minor compaction (all files) 2024-11-11T22:26:15,132 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/A in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,132 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/27f40aaddd31483b80a5e0afdcfdfda0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8393a90852c54c72ae3ee579d68d157d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/b577bb2bfea34778a340d6a9be05e9b8] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=100.7 K 2024-11-11T22:26:15,132 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,132 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/27f40aaddd31483b80a5e0afdcfdfda0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8393a90852c54c72ae3ee579d68d157d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/b577bb2bfea34778a340d6a9be05e9b8] 2024-11-11T22:26:15,133 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:15,133 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/B is initiating minor compaction (all files) 2024-11-11T22:26:15,133 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/B in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,133 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0cf48566ad6f475991d3a37a2715a8a3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/06fcf5b1047046debe33e91ecba7c6c0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d8daa3dbdb684d7196aa50e78e5f2334] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=36.6 K 2024-11-11T22:26:15,133 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27f40aaddd31483b80a5e0afdcfdfda0, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731363973082 2024-11-11T22:26:15,133 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cf48566ad6f475991d3a37a2715a8a3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731363973082 2024-11-11T22:26:15,133 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8393a90852c54c72ae3ee579d68d157d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731363973445 2024-11-11T22:26:15,134 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 06fcf5b1047046debe33e91ecba7c6c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731363973445 2024-11-11T22:26:15,134 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b577bb2bfea34778a340d6a9be05e9b8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731363974211 2024-11-11T22:26:15,134 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting d8daa3dbdb684d7196aa50e78e5f2334, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731363974211 2024-11-11T22:26:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:26:15,153 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:15,154 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-11T22:26:15,156 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:15,157 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:15,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:15,158 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#B#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:15,159 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/101c0e0ba71e40d28f1d21d07930ff61 is 50, key is test_row_0/B:col10/1731363974856/Put/seqid=0 2024-11-11T22:26:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:26:15,177 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:15,194 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111dea497d107e04e32a7a4a9334b2983c2_e47b46921ed127eb101b947b07da0745 store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:15,197 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111dea497d107e04e32a7a4a9334b2983c2_e47b46921ed127eb101b947b07da0745, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:15,197 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dea497d107e04e32a7a4a9334b2983c2_e47b46921ed127eb101b947b07da0745 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:15,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742041_1217 (size=13017) 2024-11-11T22:26:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742042_1218 (size=4469) 2024-11-11T22:26:15,223 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/101c0e0ba71e40d28f1d21d07930ff61 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/101c0e0ba71e40d28f1d21d07930ff61 2024-11-11T22:26:15,231 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e47b46921ed127eb101b947b07da0745/B of e47b46921ed127eb101b947b07da0745 into 101c0e0ba71e40d28f1d21d07930ff61(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:15,231 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:15,231 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/B, priority=13, startTime=1731363975130; duration=0sec 2024-11-11T22:26:15,231 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:15,231 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:B 2024-11-11T22:26:15,231 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:15,234 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:15,234 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): e47b46921ed127eb101b947b07da0745/C is initiating minor compaction (all files) 2024-11-11T22:26:15,234 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e47b46921ed127eb101b947b07da0745/C in TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,234 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/282c88f7f3a841f0beaff90a909d78fa, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/7f88d3b7be9d412f97a42bd52d5200f0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/18a2ba0cf479413cb5552f094efd2e32] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp, totalSize=36.6 K 2024-11-11T22:26:15,234 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 282c88f7f3a841f0beaff90a909d78fa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731363973082 2024-11-11T22:26:15,235 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f88d3b7be9d412f97a42bd52d5200f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731363973445 2024-11-11T22:26:15,235 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a2ba0cf479413cb5552f094efd2e32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731363974211 2024-11-11T22:26:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:15,237 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:15,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:26:15,250 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#C#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:15,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ef9e9422957e4a2b97fc0210fa7b343d_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363974907/Put/seqid=0 2024-11-11T22:26:15,251 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/778032a03ffb4995bb4b53faff22affd is 50, key is test_row_0/C:col10/1731363974856/Put/seqid=0 2024-11-11T22:26:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:26:15,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742044_1220 (size=13017) 2024-11-11T22:26:15,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742043_1219 (size=14994) 2024-11-11T22:26:15,294 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:15,299 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ef9e9422957e4a2b97fc0210fa7b343d_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ef9e9422957e4a2b97fc0210fa7b343d_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:15,301 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/cbeafbb8009b4a01aeed96f401f8c43e, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:15,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/cbeafbb8009b4a01aeed96f401f8c43e is 175, key is test_row_0/A:col10/1731363974907/Put/seqid=0 2024-11-11T22:26:15,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T22:26:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742045_1221 (size=39949) 2024-11-11T22:26:15,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364035328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364035334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364035443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364035453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T22:26:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:26:15,612 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e47b46921ed127eb101b947b07da0745#A#compaction#181 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:15,613 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/0656ba9fb58c4490886f965e4c872b11 is 175, key is test_row_0/A:col10/1731363974856/Put/seqid=0 2024-11-11T22:26:15,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T22:26:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742046_1222 (size=31971) 2024-11-11T22:26:15,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364035654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364035663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,698 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/778032a03ffb4995bb4b53faff22affd as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/778032a03ffb4995bb4b53faff22affd 2024-11-11T22:26:15,704 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e47b46921ed127eb101b947b07da0745/C of e47b46921ed127eb101b947b07da0745 into 778032a03ffb4995bb4b53faff22affd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:15,704 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:15,704 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/C, priority=13, startTime=1731363975132; duration=0sec 2024-11-11T22:26:15,704 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:15,704 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:C 2024-11-11T22:26:15,728 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/cbeafbb8009b4a01aeed96f401f8c43e 2024-11-11T22:26:15,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/e593613c531d41e18c25f17b4f146b20 is 50, key is test_row_0/B:col10/1731363974907/Put/seqid=0 2024-11-11T22:26:15,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742047_1223 (size=12301) 2024-11-11T22:26:15,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T22:26:15,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/e593613c531d41e18c25f17b4f146b20 2024-11-11T22:26:15,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:15,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/2958bc2c0df34abc81d67ad3f71c70f7 is 50, key is test_row_0/C:col10/1731363974907/Put/seqid=0 2024-11-11T22:26:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:26:15,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742048_1224 (size=12301) 2024-11-11T22:26:15,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T22:26:15,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:15,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:15,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:15,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35708 deadline: 1731364035973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35726 deadline: 1731364035973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:16,032 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:26:16,067 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/0656ba9fb58c4490886f965e4c872b11 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/0656ba9fb58c4490886f965e4c872b11 2024-11-11T22:26:16,072 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e47b46921ed127eb101b947b07da0745/A of e47b46921ed127eb101b947b07da0745 into 0656ba9fb58c4490886f965e4c872b11(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:16,072 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:16,072 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., storeName=e47b46921ed127eb101b947b07da0745/A, priority=13, startTime=1731363975130; duration=0sec 2024-11-11T22:26:16,073 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:16,073 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e47b46921ed127eb101b947b07da0745:A 2024-11-11T22:26:16,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T22:26:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:16,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:16,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/2958bc2c0df34abc81d67ad3f71c70f7 2024-11-11T22:26:16,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/cbeafbb8009b4a01aeed96f401f8c43e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/cbeafbb8009b4a01aeed96f401f8c43e 2024-11-11T22:26:16,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/cbeafbb8009b4a01aeed96f401f8c43e, entries=200, sequenceid=329, filesize=39.0 K 2024-11-11T22:26:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/e593613c531d41e18c25f17b4f146b20 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/e593613c531d41e18c25f17b4f146b20 2024-11-11T22:26:16,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T22:26:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:16,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:16,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:16,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:16,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/e593613c531d41e18c25f17b4f146b20, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T22:26:16,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/2958bc2c0df34abc81d67ad3f71c70f7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/2958bc2c0df34abc81d67ad3f71c70f7 2024-11-11T22:26:16,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/2958bc2c0df34abc81d67ad3f71c70f7, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T22:26:16,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e47b46921ed127eb101b947b07da0745 in 1023ms, sequenceid=329, compaction requested=false 2024-11-11T22:26:16,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:26:16,333 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-11T22:26:16,333 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:16,333 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,333 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:16,334 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-11T22:26:16,334 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:16,334 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,334 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:16,335 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-11T22:26:16,335 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:16,335 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,335 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:16,338 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-11T22:26:16,338 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:16,338 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,338 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:16,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T22:26:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:16,393 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T22:26:16,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118169f827c7ae418d8e1affb456ed9ff6_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363975325/Put/seqid=0 2024-11-11T22:26:16,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742049_1225 (size=12454) 2024-11-11T22:26:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:16,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. as already flushing 2024-11-11T22:26:16,491 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-11T22:26:16,491 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:16,491 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,491 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:16,495 INFO [Thread-666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-666. 2024-11-11T22:26:16,495 DEBUG [Thread-666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:16,495 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,495 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:16,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:16,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:16,808 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118169f827c7ae418d8e1affb456ed9ff6_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118169f827c7ae418d8e1affb456ed9ff6_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:16,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/e966dfcbf56742189ac1f7305473c9b0, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:16,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/e966dfcbf56742189ac1f7305473c9b0 is 175, key is test_row_0/A:col10/1731363975325/Put/seqid=0 2024-11-11T22:26:16,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742050_1226 (size=31255) 2024-11-11T22:26:17,213 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/e966dfcbf56742189ac1f7305473c9b0 2024-11-11T22:26:17,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/797878f823bd4858b8ea73ebcdcfce5d is 50, key is test_row_0/B:col10/1731363975325/Put/seqid=0 2024-11-11T22:26:17,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742051_1227 (size=12301) 2024-11-11T22:26:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:26:17,627 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/797878f823bd4858b8ea73ebcdcfce5d 2024-11-11T22:26:17,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/ef74656a9faf4e7494cccec51bdaa935 is 50, key is test_row_0/C:col10/1731363975325/Put/seqid=0 2024-11-11T22:26:17,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742052_1228 (size=12301) 2024-11-11T22:26:17,645 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/ef74656a9faf4e7494cccec51bdaa935 2024-11-11T22:26:17,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/e966dfcbf56742189ac1f7305473c9b0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e966dfcbf56742189ac1f7305473c9b0 2024-11-11T22:26:17,659 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e966dfcbf56742189ac1f7305473c9b0, entries=150, sequenceid=353, filesize=30.5 K 2024-11-11T22:26:17,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/797878f823bd4858b8ea73ebcdcfce5d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/797878f823bd4858b8ea73ebcdcfce5d 2024-11-11T22:26:17,666 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/797878f823bd4858b8ea73ebcdcfce5d, entries=150, sequenceid=353, filesize=12.0 K 2024-11-11T22:26:17,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/ef74656a9faf4e7494cccec51bdaa935 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/ef74656a9faf4e7494cccec51bdaa935 2024-11-11T22:26:17,671 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/ef74656a9faf4e7494cccec51bdaa935, entries=150, sequenceid=353, filesize=12.0 K 2024-11-11T22:26:17,672 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=13.42 KB/13740 for e47b46921ed127eb101b947b07da0745 in 1280ms, sequenceid=353, compaction requested=true 2024-11-11T22:26:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for e47b46921ed127eb101b947b07da0745: 2024-11-11T22:26:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-11T22:26:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-11T22:26:17,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T22:26:17,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5170 sec 2024-11-11T22:26:17,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.5220 sec 2024-11-11T22:26:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:26:19,344 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:21,486 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-11T22:26:21,486 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:21,486 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:21,487 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:21,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:25,148 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-11T22:26:25,148 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:25,148 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:25,148 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:25,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:25,177 INFO [Thread-668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-668. 2024-11-11T22:26:25,177 DEBUG [Thread-668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:25,177 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:25,178 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:25,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:25,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:26:25,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-11T22:26:25,178 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2925 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2948 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1315 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3935 rows 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1307 2024-11-11T22:26:25,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3915 rows 2024-11-11T22:26:25,179 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:26:25,179 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:26:25,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:25,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:25,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:25,181 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:26:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:26:25,186 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363985186"}]},"ts":"1731363985186"} 2024-11-11T22:26:25,188 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:26:25,188 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:26:25,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:26:25,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, UNASSIGN}] 2024-11-11T22:26:25,191 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, UNASSIGN 2024-11-11T22:26:25,192 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=e47b46921ed127eb101b947b07da0745, regionState=CLOSING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:25,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, UNASSIGN because future has completed 2024-11-11T22:26:25,193 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:26:25,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:26:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:26:25,347 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:25,347 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:26:25,347 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing e47b46921ed127eb101b947b07da0745, disabling compactions & flushes 2024-11-11T22:26:25,347 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:25,347 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:25,347 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. after waiting 0 ms 2024-11-11T22:26:25,348 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:25,348 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing e47b46921ed127eb101b947b07da0745 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T22:26:25,361 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111155d627cf2fd446b4b3c106799fd944fe_e47b46921ed127eb101b947b07da0745 is 50, key is test_row_0/A:col10/1731363981484/Put/seqid=0 2024-11-11T22:26:25,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742053_1229 (size=12454) 2024-11-11T22:26:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:26:25,767 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:25,776 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111155d627cf2fd446b4b3c106799fd944fe_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111155d627cf2fd446b4b3c106799fd944fe_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:25,778 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/59ea02cb0d73470da2b57b86964ff4f0, store: [table=TestAcidGuarantees family=A region=e47b46921ed127eb101b947b07da0745] 2024-11-11T22:26:25,778 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/59ea02cb0d73470da2b57b86964ff4f0 is 175, key is test_row_0/A:col10/1731363981484/Put/seqid=0 2024-11-11T22:26:25,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742054_1230 (size=31255) 2024-11-11T22:26:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:26:26,185 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/59ea02cb0d73470da2b57b86964ff4f0 2024-11-11T22:26:26,199 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/6fadae1bc5b24b5a8bfac1e02f71778a is 50, key is test_row_0/B:col10/1731363981484/Put/seqid=0 2024-11-11T22:26:26,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742055_1231 (size=12301) 2024-11-11T22:26:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:26:26,605 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/6fadae1bc5b24b5a8bfac1e02f71778a 2024-11-11T22:26:26,621 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/e5e50a3fe22d4c899685c39afedf8df9 is 50, key is test_row_0/C:col10/1731363981484/Put/seqid=0 2024-11-11T22:26:26,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742056_1232 (size=12301) 2024-11-11T22:26:27,028 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/e5e50a3fe22d4c899685c39afedf8df9 2024-11-11T22:26:27,039 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/A/59ea02cb0d73470da2b57b86964ff4f0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/59ea02cb0d73470da2b57b86964ff4f0 2024-11-11T22:26:27,043 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/59ea02cb0d73470da2b57b86964ff4f0, entries=150, sequenceid=361, filesize=30.5 K 2024-11-11T22:26:27,045 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/B/6fadae1bc5b24b5a8bfac1e02f71778a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/6fadae1bc5b24b5a8bfac1e02f71778a 2024-11-11T22:26:27,049 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/6fadae1bc5b24b5a8bfac1e02f71778a, entries=150, sequenceid=361, filesize=12.0 K 2024-11-11T22:26:27,050 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/.tmp/C/e5e50a3fe22d4c899685c39afedf8df9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/e5e50a3fe22d4c899685c39afedf8df9 2024-11-11T22:26:27,054 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/e5e50a3fe22d4c899685c39afedf8df9, entries=150, sequenceid=361, filesize=12.0 K 2024-11-11T22:26:27,055 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e47b46921ed127eb101b947b07da0745 in 1707ms, sequenceid=361, compaction requested=true 2024-11-11T22:26:27,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/139717c38a7b4f18bbce35d55d29bbd0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/f3f6448e721f49b9b49a8f0ac8b01f96, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/5378be40d5854d0cb262269a65441c48, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/492d686db30d499bb7c808462de542c9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/1089999c69fa47ad9ead07e174838ea8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8f48727d119645858732205bac784b9d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/489af0702b55442091a0b58507f38489, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ffea9a3a3334d80bffd94069240ec10, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/040a5bf9557a46e797fa6646a61980ac, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/476c8d23c0d2401698b37ce2ed1d79c4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e7f4f11934c94670868891784b661f1d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/d26bb4adc9c84153a3bf40c1227ad87f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/26f5783ed2b7447f8e8462592257097e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/914509cc3ca9453f8202cb41e2d80c4c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/426ea16c171b4d3e8f92decd6451734f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/628cd9433af647a18d5018ec844eeca5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/4d28f677793e4f1e94613053772c7756, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ad89e94ac1e4f25a01def8be3af0d18, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/27f40aaddd31483b80a5e0afdcfdfda0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8393a90852c54c72ae3ee579d68d157d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/b577bb2bfea34778a340d6a9be05e9b8] to archive 2024-11-11T22:26:27,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:26:27,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/139717c38a7b4f18bbce35d55d29bbd0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/139717c38a7b4f18bbce35d55d29bbd0 2024-11-11T22:26:27,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/f3f6448e721f49b9b49a8f0ac8b01f96 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/f3f6448e721f49b9b49a8f0ac8b01f96 2024-11-11T22:26:27,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/5378be40d5854d0cb262269a65441c48 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/5378be40d5854d0cb262269a65441c48 2024-11-11T22:26:27,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/492d686db30d499bb7c808462de542c9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/492d686db30d499bb7c808462de542c9 2024-11-11T22:26:27,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/1089999c69fa47ad9ead07e174838ea8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/1089999c69fa47ad9ead07e174838ea8 2024-11-11T22:26:27,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8f48727d119645858732205bac784b9d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8f48727d119645858732205bac784b9d 2024-11-11T22:26:27,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/489af0702b55442091a0b58507f38489 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/489af0702b55442091a0b58507f38489 2024-11-11T22:26:27,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ffea9a3a3334d80bffd94069240ec10 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ffea9a3a3334d80bffd94069240ec10 2024-11-11T22:26:27,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/040a5bf9557a46e797fa6646a61980ac to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/040a5bf9557a46e797fa6646a61980ac 2024-11-11T22:26:27,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/476c8d23c0d2401698b37ce2ed1d79c4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/476c8d23c0d2401698b37ce2ed1d79c4 2024-11-11T22:26:27,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e7f4f11934c94670868891784b661f1d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e7f4f11934c94670868891784b661f1d 2024-11-11T22:26:27,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/d26bb4adc9c84153a3bf40c1227ad87f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/d26bb4adc9c84153a3bf40c1227ad87f 2024-11-11T22:26:27,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/26f5783ed2b7447f8e8462592257097e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/26f5783ed2b7447f8e8462592257097e 2024-11-11T22:26:27,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/914509cc3ca9453f8202cb41e2d80c4c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/914509cc3ca9453f8202cb41e2d80c4c 2024-11-11T22:26:27,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/426ea16c171b4d3e8f92decd6451734f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/426ea16c171b4d3e8f92decd6451734f 2024-11-11T22:26:27,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/628cd9433af647a18d5018ec844eeca5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/628cd9433af647a18d5018ec844eeca5 2024-11-11T22:26:27,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/4d28f677793e4f1e94613053772c7756 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/4d28f677793e4f1e94613053772c7756 2024-11-11T22:26:27,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ad89e94ac1e4f25a01def8be3af0d18 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8ad89e94ac1e4f25a01def8be3af0d18 2024-11-11T22:26:27,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/27f40aaddd31483b80a5e0afdcfdfda0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/27f40aaddd31483b80a5e0afdcfdfda0 2024-11-11T22:26:27,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8393a90852c54c72ae3ee579d68d157d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/8393a90852c54c72ae3ee579d68d157d 2024-11-11T22:26:27,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/b577bb2bfea34778a340d6a9be05e9b8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/b577bb2bfea34778a340d6a9be05e9b8 2024-11-11T22:26:27,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/226d32248ee2447d99a1a13ad5024863, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d1386f320408416ba13b3eb42c8b7b0a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/fc23dc4d27414267b3a0a9254b868c5a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/e6a6d5f9883d43bebea545323b665c02, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/48171f62e12643d2acbaafc1d2f84202, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0d2b5c2579654975874ceeb8cae0dde5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8a969df0952f434e9f35bbdf02facdd8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/68ad88bf8f76419c8c51aa55cbe2795e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c4f86b60970040b5a2dee72fcb52db89, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/84ec6db24bb24a4281f7807b0bbd7375, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/a1586540c4734b1b9227b860c8d4c9da, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c57b9fb7233a44178d600a00a6b69a33, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8449b8b63b77493dad5bef6f740aa8ac, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/74459a62b2ef46f89268c747816de850, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/b630bc44187a447dbea8ebab98656f2e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/5350eea2c0524036b06c7d2589e5124d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/6983cd0d3b344c7394fb6efec54acc72, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0cf48566ad6f475991d3a37a2715a8a3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/57db733ca2124e41a8e685c1c528def2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/06fcf5b1047046debe33e91ecba7c6c0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d8daa3dbdb684d7196aa50e78e5f2334] to archive 2024-11-11T22:26:27,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:26:27,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/226d32248ee2447d99a1a13ad5024863 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/226d32248ee2447d99a1a13ad5024863 2024-11-11T22:26:27,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d1386f320408416ba13b3eb42c8b7b0a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d1386f320408416ba13b3eb42c8b7b0a 2024-11-11T22:26:27,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/fc23dc4d27414267b3a0a9254b868c5a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/fc23dc4d27414267b3a0a9254b868c5a 2024-11-11T22:26:27,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/e6a6d5f9883d43bebea545323b665c02 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/e6a6d5f9883d43bebea545323b665c02 2024-11-11T22:26:27,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/48171f62e12643d2acbaafc1d2f84202 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/48171f62e12643d2acbaafc1d2f84202 2024-11-11T22:26:27,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0d2b5c2579654975874ceeb8cae0dde5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0d2b5c2579654975874ceeb8cae0dde5 2024-11-11T22:26:27,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8a969df0952f434e9f35bbdf02facdd8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8a969df0952f434e9f35bbdf02facdd8 2024-11-11T22:26:27,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/68ad88bf8f76419c8c51aa55cbe2795e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/68ad88bf8f76419c8c51aa55cbe2795e 2024-11-11T22:26:27,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c4f86b60970040b5a2dee72fcb52db89 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c4f86b60970040b5a2dee72fcb52db89 2024-11-11T22:26:27,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/84ec6db24bb24a4281f7807b0bbd7375 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/84ec6db24bb24a4281f7807b0bbd7375 2024-11-11T22:26:27,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/a1586540c4734b1b9227b860c8d4c9da to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/a1586540c4734b1b9227b860c8d4c9da 2024-11-11T22:26:27,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c57b9fb7233a44178d600a00a6b69a33 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/c57b9fb7233a44178d600a00a6b69a33 2024-11-11T22:26:27,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8449b8b63b77493dad5bef6f740aa8ac to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/8449b8b63b77493dad5bef6f740aa8ac 2024-11-11T22:26:27,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/74459a62b2ef46f89268c747816de850 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/74459a62b2ef46f89268c747816de850 2024-11-11T22:26:27,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/b630bc44187a447dbea8ebab98656f2e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/b630bc44187a447dbea8ebab98656f2e 2024-11-11T22:26:27,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/5350eea2c0524036b06c7d2589e5124d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/5350eea2c0524036b06c7d2589e5124d 2024-11-11T22:26:27,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/6983cd0d3b344c7394fb6efec54acc72 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/6983cd0d3b344c7394fb6efec54acc72 2024-11-11T22:26:27,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0cf48566ad6f475991d3a37a2715a8a3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/0cf48566ad6f475991d3a37a2715a8a3 2024-11-11T22:26:27,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/57db733ca2124e41a8e685c1c528def2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/57db733ca2124e41a8e685c1c528def2 2024-11-11T22:26:27,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/06fcf5b1047046debe33e91ecba7c6c0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/06fcf5b1047046debe33e91ecba7c6c0 2024-11-11T22:26:27,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d8daa3dbdb684d7196aa50e78e5f2334 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/d8daa3dbdb684d7196aa50e78e5f2334 2024-11-11T22:26:27,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/1f2b500d1f8d490ca202d8fad1f4e5c3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9cbbf887512d44d6872083b9b3c13e39, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/3688e6fad17c4589b1f03e850db94c72, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4719d4b17565458f94f22649d54843ff, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/91207489620b4e69bc9d8290b1261eb5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/54c7faf9001846578658c0367579d019, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/163e7c53019a4e05a101900d7adabd2f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/73f44159d07f4edf80116c53b0a73c88, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9ce9fdbfe9e44f57986a57486417bc27, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/89aba986189f4cfe927290c87aaf3879, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/67f6d9beb1494e6eb2a5b9f20277033b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4187fb40962f40cf9cdc29409faa159e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/665c6a204d934821a9210caadfe7e235, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/de5aeac813424d6db51475e3217a83b3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/40a28dda743244db8d4256cc65a81139, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/ef475e61a41c4b7fb8b51169ba2b6fd7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/124ca945df2b4d92b9065b34e05713e2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/282c88f7f3a841f0beaff90a909d78fa, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/b723008ad14b46e3b00d15cc747660a9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/7f88d3b7be9d412f97a42bd52d5200f0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/18a2ba0cf479413cb5552f094efd2e32] to archive 2024-11-11T22:26:27,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:26:27,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/1f2b500d1f8d490ca202d8fad1f4e5c3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/1f2b500d1f8d490ca202d8fad1f4e5c3 2024-11-11T22:26:27,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9cbbf887512d44d6872083b9b3c13e39 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9cbbf887512d44d6872083b9b3c13e39 2024-11-11T22:26:27,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/3688e6fad17c4589b1f03e850db94c72 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/3688e6fad17c4589b1f03e850db94c72 2024-11-11T22:26:27,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4719d4b17565458f94f22649d54843ff to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4719d4b17565458f94f22649d54843ff 2024-11-11T22:26:27,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/91207489620b4e69bc9d8290b1261eb5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/91207489620b4e69bc9d8290b1261eb5 2024-11-11T22:26:27,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/54c7faf9001846578658c0367579d019 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/54c7faf9001846578658c0367579d019 2024-11-11T22:26:27,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/163e7c53019a4e05a101900d7adabd2f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/163e7c53019a4e05a101900d7adabd2f 2024-11-11T22:26:27,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/73f44159d07f4edf80116c53b0a73c88 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/73f44159d07f4edf80116c53b0a73c88 2024-11-11T22:26:27,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9ce9fdbfe9e44f57986a57486417bc27 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/9ce9fdbfe9e44f57986a57486417bc27 2024-11-11T22:26:27,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/89aba986189f4cfe927290c87aaf3879 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/89aba986189f4cfe927290c87aaf3879 2024-11-11T22:26:27,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/67f6d9beb1494e6eb2a5b9f20277033b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/67f6d9beb1494e6eb2a5b9f20277033b 2024-11-11T22:26:27,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4187fb40962f40cf9cdc29409faa159e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/4187fb40962f40cf9cdc29409faa159e 2024-11-11T22:26:27,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/665c6a204d934821a9210caadfe7e235 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/665c6a204d934821a9210caadfe7e235 2024-11-11T22:26:27,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/de5aeac813424d6db51475e3217a83b3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/de5aeac813424d6db51475e3217a83b3 2024-11-11T22:26:27,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/40a28dda743244db8d4256cc65a81139 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/40a28dda743244db8d4256cc65a81139 2024-11-11T22:26:27,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/ef475e61a41c4b7fb8b51169ba2b6fd7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/ef475e61a41c4b7fb8b51169ba2b6fd7 2024-11-11T22:26:27,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/124ca945df2b4d92b9065b34e05713e2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/124ca945df2b4d92b9065b34e05713e2 2024-11-11T22:26:27,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/282c88f7f3a841f0beaff90a909d78fa to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/282c88f7f3a841f0beaff90a909d78fa 2024-11-11T22:26:27,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/b723008ad14b46e3b00d15cc747660a9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/b723008ad14b46e3b00d15cc747660a9 2024-11-11T22:26:27,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/7f88d3b7be9d412f97a42bd52d5200f0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/7f88d3b7be9d412f97a42bd52d5200f0 2024-11-11T22:26:27,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/18a2ba0cf479413cb5552f094efd2e32 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/18a2ba0cf479413cb5552f094efd2e32 2024-11-11T22:26:27,124 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/recovered.edits/364.seqid, newMaxSeqId=364, maxSeqId=4 2024-11-11T22:26:27,124 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. 2024-11-11T22:26:27,124 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for e47b46921ed127eb101b947b07da0745: Waiting for close lock at 1731363985347Running coprocessor pre-close hooks at 1731363985347Disabling compacts and flushes for region at 1731363985347Disabling writes for close at 1731363985347Obtaining lock to block concurrent updates at 1731363985348 (+1 ms)Preparing flush snapshotting stores in e47b46921ed127eb101b947b07da0745 at 1731363985348Finished memstore snapshotting TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731363985349 (+1 ms)Flushing stores of TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745. at 1731363985350 (+1 ms)Flushing e47b46921ed127eb101b947b07da0745/A: creating writer at 1731363985350Flushing mob file e47b46921ed127eb101b947b07da0745/A: appending metadata at 1731363985361 (+11 ms)Flushing mob file e47b46921ed127eb101b947b07da0745/A: closing flushed file at 1731363985361Flushing e47b46921ed127eb101b947b07da0745/A: appending metadata at 1731363985778 (+417 ms)Flushing e47b46921ed127eb101b947b07da0745/A: closing flushed file at 1731363985778Flushing e47b46921ed127eb101b947b07da0745/B: creating writer at 1731363986195 (+417 ms)Flushing e47b46921ed127eb101b947b07da0745/B: appending metadata at 1731363986198 (+3 ms)Flushing e47b46921ed127eb101b947b07da0745/B: closing flushed file at 1731363986199 (+1 ms)Flushing e47b46921ed127eb101b947b07da0745/C: creating writer at 1731363986614 (+415 ms)Flushing e47b46921ed127eb101b947b07da0745/C: appending metadata at 1731363986620 (+6 ms)Flushing e47b46921ed127eb101b947b07da0745/C: closing flushed file at 1731363986620Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28e5b9fe: reopening flushed file at 1731363987037 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32ecdcc9: reopening flushed file at 1731363987044 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c5b82cb: reopening flushed file at 1731363987049 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e47b46921ed127eb101b947b07da0745 in 1707ms, sequenceid=361, compaction requested=true at 1731363987055 (+6 ms)Writing region close event to WAL at 1731363987121 (+66 ms)Running coprocessor post-close hooks at 1731363987124 (+3 ms)Closed at 1731363987124 2024-11-11T22:26:27,126 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,126 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=e47b46921ed127eb101b947b07da0745, regionState=CLOSED 2024-11-11T22:26:27,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:26:27,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-11T22:26:27,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure e47b46921ed127eb101b947b07da0745, server=8d82b89dd27d,39613,1731363918313 in 1.9350 sec 2024-11-11T22:26:27,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-11T22:26:27,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e47b46921ed127eb101b947b07da0745, UNASSIGN in 1.9400 sec 2024-11-11T22:26:27,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T22:26:27,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9420 sec 2024-11-11T22:26:27,133 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363987133"}]},"ts":"1731363987133"} 2024-11-11T22:26:27,134 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:26:27,134 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:26:27,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9530 sec 2024-11-11T22:26:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:26:27,335 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:27,336 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:26:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:27,340 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T22:26:27,341 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:27,348 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,354 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/recovered.edits] 2024-11-11T22:26:27,360 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/0656ba9fb58c4490886f965e4c872b11 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/0656ba9fb58c4490886f965e4c872b11 2024-11-11T22:26:27,361 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/59ea02cb0d73470da2b57b86964ff4f0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/59ea02cb0d73470da2b57b86964ff4f0 2024-11-11T22:26:27,362 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/cbeafbb8009b4a01aeed96f401f8c43e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/cbeafbb8009b4a01aeed96f401f8c43e 2024-11-11T22:26:27,364 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e966dfcbf56742189ac1f7305473c9b0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/A/e966dfcbf56742189ac1f7305473c9b0 2024-11-11T22:26:27,366 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/101c0e0ba71e40d28f1d21d07930ff61 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/101c0e0ba71e40d28f1d21d07930ff61 2024-11-11T22:26:27,367 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/6fadae1bc5b24b5a8bfac1e02f71778a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/6fadae1bc5b24b5a8bfac1e02f71778a 2024-11-11T22:26:27,368 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/797878f823bd4858b8ea73ebcdcfce5d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/797878f823bd4858b8ea73ebcdcfce5d 2024-11-11T22:26:27,369 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/e593613c531d41e18c25f17b4f146b20 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/B/e593613c531d41e18c25f17b4f146b20 2024-11-11T22:26:27,371 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/2958bc2c0df34abc81d67ad3f71c70f7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/2958bc2c0df34abc81d67ad3f71c70f7 2024-11-11T22:26:27,372 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/778032a03ffb4995bb4b53faff22affd to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/778032a03ffb4995bb4b53faff22affd 2024-11-11T22:26:27,373 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/e5e50a3fe22d4c899685c39afedf8df9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/e5e50a3fe22d4c899685c39afedf8df9 2024-11-11T22:26:27,374 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/ef74656a9faf4e7494cccec51bdaa935 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/C/ef74656a9faf4e7494cccec51bdaa935 2024-11-11T22:26:27,376 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/recovered.edits/364.seqid to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745/recovered.edits/364.seqid 2024-11-11T22:26:27,377 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,377 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:26:27,377 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:26:27,378 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T22:26:27,381 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110da1920e204240048a239a6731f6d65f_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110da1920e204240048a239a6731f6d65f_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,383 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111136c0c35072ce434db7e9338553476fdb_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111136c0c35072ce434db7e9338553476fdb_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,384 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111451c35267ee54a0e9d5cee7bb2371c34_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111451c35267ee54a0e9d5cee7bb2371c34_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,385 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111155d627cf2fd446b4b3c106799fd944fe_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111155d627cf2fd446b4b3c106799fd944fe_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,386 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111156a95a6f78164797a5c6c6a9e63f1624_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111156a95a6f78164797a5c6c6a9e63f1624_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,387 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116ea49f8f22994952b463eb5962d017d1_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116ea49f8f22994952b463eb5962d017d1_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,388 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118169f827c7ae418d8e1affb456ed9ff6_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118169f827c7ae418d8e1affb456ed9ff6_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,389 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111851b4d710cb343f3b1c7087c8d2e588c_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111851b4d710cb343f3b1c7087c8d2e588c_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,390 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111995181acc25d4433a0b13090cd1afa98_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111995181acc25d4433a0b13090cd1afa98_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,391 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a7268523b7514b57b4938b8f317b7630_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a7268523b7514b57b4938b8f317b7630_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,392 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a7326106fa22413fb3f12e652618b86b_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a7326106fa22413fb3f12e652618b86b_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,393 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111acdd6d7d1a2f48ca9007c7ea76cba0da_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111acdd6d7d1a2f48ca9007c7ea76cba0da_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,394 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111afb896c3c0474822b69dded484b27f46_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111afb896c3c0474822b69dded484b27f46_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,395 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b8672a8180d3492a8077b0a0386e2738_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b8672a8180d3492a8077b0a0386e2738_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,395 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c9486fb48e2a4ccbad033937c481dbb0_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c9486fb48e2a4ccbad033937c481dbb0_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,396 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ca41fb97c9bd4851b0c5e44b634f7f00_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ca41fb97c9bd4851b0c5e44b634f7f00_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,397 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0c9ad391801463884f275a71058e914_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0c9ad391801463884f275a71058e914_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,398 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ebdcbc1554394ade9985b134af72e06e_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ebdcbc1554394ade9985b134af72e06e_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,399 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ef9e9422957e4a2b97fc0210fa7b343d_e47b46921ed127eb101b947b07da0745 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ef9e9422957e4a2b97fc0210fa7b343d_e47b46921ed127eb101b947b07da0745 2024-11-11T22:26:27,399 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:26:27,402 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:27,404 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:26:27,406 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:26:27,407 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:27,407 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:26:27,407 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731363987407"}]},"ts":"9223372036854775807"} 2024-11-11T22:26:27,409 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:26:27,409 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e47b46921ed127eb101b947b07da0745, NAME => 'TestAcidGuarantees,,1731363953123.e47b46921ed127eb101b947b07da0745.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:26:27,410 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:26:27,410 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731363987410"}]},"ts":"9223372036854775807"} 2024-11-11T22:26:27,411 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:26:27,412 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:27,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 76 msec 2024-11-11T22:26:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T22:26:27,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:26:27,443 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:27,453 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_437819696_22 at /127.0.0.1:49252 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/cluster_b777580f-683f-c355-fb6d-abef9c123800/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/cluster_b777580f-683f-c355-fb6d-abef9c123800/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 579), ProcessCount=11 (was 11), AvailableMemoryMB=5710 (was 6511) 2024-11-11T22:26:27,461 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=5709 2024-11-11T22:26:27,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:26:27,463 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:26:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:27,464 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:26:27,465 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:27,465 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-11T22:26:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-11T22:26:27,465 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:26:27,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742057_1233 (size=959) 2024-11-11T22:26:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-11T22:26:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-11T22:26:27,876 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e 2024-11-11T22:26:27,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742058_1234 (size=53) 2024-11-11T22:26:27,919 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T22:26:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-11T22:26:28,291 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:26:28,291 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8b5cc6a2f2e03a352bea5a20a4c912ce, disabling compactions & flushes 2024-11-11T22:26:28,291 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:28,291 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:28,291 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. after waiting 0 ms 2024-11-11T22:26:28,291 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:28,291 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:28,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: Waiting for close lock at 1731363988291Disabling compacts and flushes for region at 1731363988291Disabling writes for close at 1731363988291Writing region close event to WAL at 1731363988291Closed at 1731363988291 2024-11-11T22:26:28,294 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:26:28,295 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731363988295"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731363988295"}]},"ts":"1731363988295"} 2024-11-11T22:26:28,300 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:26:28,301 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:26:28,301 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363988301"}]},"ts":"1731363988301"} 2024-11-11T22:26:28,302 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:26:28,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b5cc6a2f2e03a352bea5a20a4c912ce, ASSIGN}] 2024-11-11T22:26:28,303 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b5cc6a2f2e03a352bea5a20a4c912ce, ASSIGN 2024-11-11T22:26:28,304 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b5cc6a2f2e03a352bea5a20a4c912ce, ASSIGN; state=OFFLINE, location=8d82b89dd27d,39613,1731363918313; forceNewPlan=false, retain=false 2024-11-11T22:26:28,455 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=8b5cc6a2f2e03a352bea5a20a4c912ce, regionState=OPENING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:28,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b5cc6a2f2e03a352bea5a20a4c912ce, ASSIGN because future has completed 2024-11-11T22:26:28,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:26:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-11T22:26:28,619 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:28,619 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:26:28,620 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,620 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:26:28,621 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,621 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,624 INFO [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,626 INFO [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b5cc6a2f2e03a352bea5a20a4c912ce columnFamilyName A 2024-11-11T22:26:28,626 DEBUG [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:28,627 INFO [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] regionserver.HStore(327): Store=8b5cc6a2f2e03a352bea5a20a4c912ce/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:26:28,627 INFO [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,630 INFO [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b5cc6a2f2e03a352bea5a20a4c912ce columnFamilyName B 2024-11-11T22:26:28,630 DEBUG [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:28,630 INFO [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] regionserver.HStore(327): Store=8b5cc6a2f2e03a352bea5a20a4c912ce/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:26:28,631 INFO [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,633 INFO [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b5cc6a2f2e03a352bea5a20a4c912ce columnFamilyName C 2024-11-11T22:26:28,633 DEBUG [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:28,634 INFO [StoreOpener-8b5cc6a2f2e03a352bea5a20a4c912ce-1 {}] regionserver.HStore(327): Store=8b5cc6a2f2e03a352bea5a20a4c912ce/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:26:28,634 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,635 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,636 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,637 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,637 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,638 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:26:28,640 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,642 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:26:28,643 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 8b5cc6a2f2e03a352bea5a20a4c912ce; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63305865, jitterRate=-0.05666910111904144}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:26:28,643 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:28,644 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: Running coprocessor pre-open hook at 1731363988621Writing region info on filesystem at 1731363988621Initializing all the Stores at 1731363988623 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363988623Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363988623Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363988623Cleaning up temporary data from old regions at 1731363988637 (+14 ms)Running coprocessor post-open hooks at 1731363988643 (+6 ms)Region opened successfully at 1731363988644 (+1 ms) 2024-11-11T22:26:28,645 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., pid=61, masterSystemTime=1731363988613 2024-11-11T22:26:28,647 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:28,647 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:28,648 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=8b5cc6a2f2e03a352bea5a20a4c912ce, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:28,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:26:28,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T22:26:28,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 in 190 msec 2024-11-11T22:26:28,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-11T22:26:28,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b5cc6a2f2e03a352bea5a20a4c912ce, ASSIGN in 350 msec 2024-11-11T22:26:28,654 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:26:28,654 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363988654"}]},"ts":"1731363988654"} 2024-11-11T22:26:28,656 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:26:28,656 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:26:28,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1930 sec 2024-11-11T22:26:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-11T22:26:29,615 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:29,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,618 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,621 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@475109e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,625 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@506d6437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,626 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,629 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:26:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,40133,1731363917560 2024-11-11T22:26:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d53ccca 2024-11-11T22:26:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:26:29,631 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:26:29,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,634 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b48c69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,635 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1506fea7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,638 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ac4719, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,640 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b04c6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,642 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26669f08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,644 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b2b5fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,647 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429922b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,649 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736999fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,651 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51aae8c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,652 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73aa5537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,654 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,654 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9df3a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20299872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,657 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5744d074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,659 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b00c4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,661 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111aa29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,662 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c7c21a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,664 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a7ab4d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,666 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b98bf96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:29,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:29,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@104293dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:29,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f2e862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:29,670 DEBUG [Thread-1100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,670 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,670 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,671 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,671 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,671 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-11T22:26:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,672 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,673 DEBUG [Thread-1096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,673 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T22:26:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:26:29,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,675 DEBUG [Thread-1098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,675 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:26:29,675 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:26:29,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:26:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:26:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:29,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:26:29,679 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:29,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:29,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:26:29,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:26:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:26:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:26:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:29,686 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:29,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:26:29,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/fc321eda01a840eb81e68340f06b712f is 50, key is test_row_0/A:col10/1731363989684/Put/seqid=0 2024-11-11T22:26:29,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:29,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364049713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364049712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:29,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364049714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:29,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364049716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:29,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364049719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:29,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742059_1235 (size=16681) 2024-11-11T22:26:29,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/fc321eda01a840eb81e68340f06b712f 2024-11-11T22:26:29,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/08f0128870bb4a3bbaa9cd65057aaaed is 50, key is test_row_0/B:col10/1731363989684/Put/seqid=0 2024-11-11T22:26:29,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742060_1236 (size=12001) 2024-11-11T22:26:29,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/08f0128870bb4a3bbaa9cd65057aaaed 2024-11-11T22:26:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T22:26:29,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/f8e9480239c846e0ade4aff91c2b12e8 is 50, key is test_row_0/C:col10/1731363989684/Put/seqid=0 2024-11-11T22:26:29,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742061_1237 (size=12001) 2024-11-11T22:26:29,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364049823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:29,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364049823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:29,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364049824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:29,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364049824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:29,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T22:26:29,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:29,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:29,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:29,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:29,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364049833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:29,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T22:26:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:29,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T22:26:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364050033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364050033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364050033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364050033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364050043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T22:26:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:30,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:30,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/f8e9480239c846e0ade4aff91c2b12e8 2024-11-11T22:26:30,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/fc321eda01a840eb81e68340f06b712f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/fc321eda01a840eb81e68340f06b712f 2024-11-11T22:26:30,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/fc321eda01a840eb81e68340f06b712f, entries=250, sequenceid=13, filesize=16.3 K 2024-11-11T22:26:30,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/08f0128870bb4a3bbaa9cd65057aaaed as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/08f0128870bb4a3bbaa9cd65057aaaed 2024-11-11T22:26:30,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/08f0128870bb4a3bbaa9cd65057aaaed, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T22:26:30,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/f8e9480239c846e0ade4aff91c2b12e8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/f8e9480239c846e0ade4aff91c2b12e8 2024-11-11T22:26:30,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/f8e9480239c846e0ade4aff91c2b12e8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T22:26:30,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 540ms, sequenceid=13, compaction requested=false 2024-11-11T22:26:30,226 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T22:26:30,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:30,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T22:26:30,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:30,292 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:26:30,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/8155ecdaef5541d4a8c4aeea5656c2cc is 50, key is test_row_0/A:col10/1731363989711/Put/seqid=0 2024-11-11T22:26:30,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742062_1238 (size=12001) 2024-11-11T22:26:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T22:26:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:30,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364050349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364050350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364050350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364050353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364050353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364050463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364050463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364050463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364050463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364050673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364050673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364050673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364050673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,700 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/8155ecdaef5541d4a8c4aeea5656c2cc 2024-11-11T22:26:30,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/8b1cb0dafdea4fd6bdc6c691e7cc79ed is 50, key is test_row_0/B:col10/1731363989711/Put/seqid=0 2024-11-11T22:26:30,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742063_1239 (size=12001) 2024-11-11T22:26:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T22:26:30,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364050863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364050983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364050983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364050983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364050983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:31,111 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/8b1cb0dafdea4fd6bdc6c691e7cc79ed 2024-11-11T22:26:31,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/259cc43ff0594061b0735342aae47e2f is 50, key is test_row_0/C:col10/1731363989711/Put/seqid=0 2024-11-11T22:26:31,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742064_1240 (size=12001) 2024-11-11T22:26:31,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364051493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:31,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364051493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:31,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364051493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:31,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364051493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:31,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:31,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:31,526 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/259cc43ff0594061b0735342aae47e2f 2024-11-11T22:26:31,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/8155ecdaef5541d4a8c4aeea5656c2cc as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/8155ecdaef5541d4a8c4aeea5656c2cc 2024-11-11T22:26:31,535 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/8155ecdaef5541d4a8c4aeea5656c2cc, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T22:26:31,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/8b1cb0dafdea4fd6bdc6c691e7cc79ed as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/8b1cb0dafdea4fd6bdc6c691e7cc79ed 2024-11-11T22:26:31,539 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/8b1cb0dafdea4fd6bdc6c691e7cc79ed, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T22:26:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/259cc43ff0594061b0735342aae47e2f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/259cc43ff0594061b0735342aae47e2f 2024-11-11T22:26:31,543 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/259cc43ff0594061b0735342aae47e2f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T22:26:31,544 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 1252ms, sequenceid=37, compaction requested=false 2024-11-11T22:26:31,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:31,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:31,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-11T22:26:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-11T22:26:31,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T22:26:31,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-11-11T22:26:31,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.8760 sec 2024-11-11T22:26:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T22:26:31,833 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:31,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-11T22:26:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T22:26:31,835 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:31,836 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:31,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:31,876 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:31,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T22:26:31,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/255ec97985b54439bcef81019cc47e84 is 50, key is test_row_0/A:col10/1731363990351/Put/seqid=0 2024-11-11T22:26:31,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742065_1241 (size=14341) 2024-11-11T22:26:31,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T22:26:31,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364051952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:31,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T22:26:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:31,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364052063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:32,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T22:26:32,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:32,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T22:26:32,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:32,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364052273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:32,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/255ec97985b54439bcef81019cc47e84 2024-11-11T22:26:32,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/f233c7ec89d14451aecdd9702ca26b90 is 50, key is test_row_0/B:col10/1731363990351/Put/seqid=0 2024-11-11T22:26:32,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T22:26:32,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:32,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742066_1242 (size=12001) 2024-11-11T22:26:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T22:26:32,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:32,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T22:26:32,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364052503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:32,506 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:32,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364052513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:32,517 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:32,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364052523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:32,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364052523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:32,527 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:32,528 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:32,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364052583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:32,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T22:26:32,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:32,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,650 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:26:32,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/f233c7ec89d14451aecdd9702ca26b90 2024-11-11T22:26:32,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/bd5b462f3e214d2caf2bd10b40473ded is 50, key is test_row_0/C:col10/1731363990351/Put/seqid=0 2024-11-11T22:26:32,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742067_1243 (size=12001) 2024-11-11T22:26:32,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T22:26:32,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:32,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T22:26:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:32,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T22:26:33,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T22:26:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:33,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:33,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364053092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:33,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/bd5b462f3e214d2caf2bd10b40473ded 2024-11-11T22:26:33,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/255ec97985b54439bcef81019cc47e84 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/255ec97985b54439bcef81019cc47e84 2024-11-11T22:26:33,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/255ec97985b54439bcef81019cc47e84, entries=200, sequenceid=50, filesize=14.0 K 2024-11-11T22:26:33,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/f233c7ec89d14451aecdd9702ca26b90 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f233c7ec89d14451aecdd9702ca26b90 2024-11-11T22:26:33,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f233c7ec89d14451aecdd9702ca26b90, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T22:26:33,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/bd5b462f3e214d2caf2bd10b40473ded as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bd5b462f3e214d2caf2bd10b40473ded 2024-11-11T22:26:33,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bd5b462f3e214d2caf2bd10b40473ded, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T22:26:33,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 1253ms, sequenceid=50, compaction requested=true 2024-11-11T22:26:33,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:33,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:33,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:33,130 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:33,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:33,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:33,130 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:33,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:33,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:33,131 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:33,131 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:33,131 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/A is initiating minor compaction (all files) 2024-11-11T22:26:33,131 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/B is initiating minor compaction (all files) 2024-11-11T22:26:33,131 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/A in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:33,131 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/B in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:33,131 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/fc321eda01a840eb81e68340f06b712f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/8155ecdaef5541d4a8c4aeea5656c2cc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/255ec97985b54439bcef81019cc47e84] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=42.0 K 2024-11-11T22:26:33,131 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/08f0128870bb4a3bbaa9cd65057aaaed, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/8b1cb0dafdea4fd6bdc6c691e7cc79ed, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f233c7ec89d14451aecdd9702ca26b90] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=35.2 K 2024-11-11T22:26:33,131 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc321eda01a840eb81e68340f06b712f, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363989682 2024-11-11T22:26:33,131 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 08f0128870bb4a3bbaa9cd65057aaaed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363989684 2024-11-11T22:26:33,132 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8155ecdaef5541d4a8c4aeea5656c2cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731363989711 2024-11-11T22:26:33,133 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b1cb0dafdea4fd6bdc6c691e7cc79ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731363989711 2024-11-11T22:26:33,133 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 255ec97985b54439bcef81019cc47e84, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731363990347 2024-11-11T22:26:33,133 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f233c7ec89d14451aecdd9702ca26b90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731363990349 2024-11-11T22:26:33,146 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#A#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:33,146 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#B#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:33,147 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/b8b76d8baade4adfbb7208fec7fdccf8 is 50, key is test_row_0/B:col10/1731363990351/Put/seqid=0 2024-11-11T22:26:33,147 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/15fd28e050684e3ebdb923c5709a0d06 is 50, key is test_row_0/A:col10/1731363990351/Put/seqid=0 2024-11-11T22:26:33,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742068_1244 (size=12104) 2024-11-11T22:26:33,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742069_1245 (size=12104) 2024-11-11T22:26:33,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T22:26:33,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:33,211 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:26:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/b9e354f6f8364d079c018cdb1f4950d9 is 50, key is test_row_0/A:col10/1731363991951/Put/seqid=0 2024-11-11T22:26:33,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742070_1246 (size=12001) 2024-11-11T22:26:33,560 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/15fd28e050684e3ebdb923c5709a0d06 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/15fd28e050684e3ebdb923c5709a0d06 2024-11-11T22:26:33,560 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/b8b76d8baade4adfbb7208fec7fdccf8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b8b76d8baade4adfbb7208fec7fdccf8 2024-11-11T22:26:33,565 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/B of 8b5cc6a2f2e03a352bea5a20a4c912ce into b8b76d8baade4adfbb7208fec7fdccf8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:33,565 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/A of 8b5cc6a2f2e03a352bea5a20a4c912ce into 15fd28e050684e3ebdb923c5709a0d06(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:33,565 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:33,565 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:33,565 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/B, priority=13, startTime=1731363993130; duration=0sec 2024-11-11T22:26:33,565 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/A, priority=13, startTime=1731363993130; duration=0sec 2024-11-11T22:26:33,565 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:33,565 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:B 2024-11-11T22:26:33,565 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:33,565 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:A 2024-11-11T22:26:33,565 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:33,566 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:33,566 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/C is initiating minor compaction (all files) 2024-11-11T22:26:33,566 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/C in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:33,566 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/f8e9480239c846e0ade4aff91c2b12e8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/259cc43ff0594061b0735342aae47e2f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bd5b462f3e214d2caf2bd10b40473ded] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=35.2 K 2024-11-11T22:26:33,566 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e9480239c846e0ade4aff91c2b12e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363989684 2024-11-11T22:26:33,567 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 259cc43ff0594061b0735342aae47e2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731363989711 2024-11-11T22:26:33,567 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting bd5b462f3e214d2caf2bd10b40473ded, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731363990349 2024-11-11T22:26:33,573 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#C#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:33,574 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/7e1f5115410c4dd18797c23b6d2663ab is 50, key is test_row_0/C:col10/1731363990351/Put/seqid=0 2024-11-11T22:26:33,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742071_1247 (size=12104) 2024-11-11T22:26:33,620 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/b9e354f6f8364d079c018cdb1f4950d9 2024-11-11T22:26:33,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/43599986061a401fa3f33dec77980910 is 50, key is test_row_0/B:col10/1731363991951/Put/seqid=0 2024-11-11T22:26:33,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742072_1248 (size=12001) 2024-11-11T22:26:33,630 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/43599986061a401fa3f33dec77980910 2024-11-11T22:26:33,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/5ce8741feb2e4fdcacb968e5e50a0e83 is 50, key is test_row_0/C:col10/1731363991951/Put/seqid=0 2024-11-11T22:26:33,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742073_1249 (size=12001) 2024-11-11T22:26:33,658 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/5ce8741feb2e4fdcacb968e5e50a0e83 2024-11-11T22:26:33,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/b9e354f6f8364d079c018cdb1f4950d9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b9e354f6f8364d079c018cdb1f4950d9 2024-11-11T22:26:33,667 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b9e354f6f8364d079c018cdb1f4950d9, entries=150, sequenceid=73, filesize=11.7 K 2024-11-11T22:26:33,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/43599986061a401fa3f33dec77980910 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/43599986061a401fa3f33dec77980910 2024-11-11T22:26:33,672 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/43599986061a401fa3f33dec77980910, entries=150, sequenceid=73, filesize=11.7 K 2024-11-11T22:26:33,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/5ce8741feb2e4fdcacb968e5e50a0e83 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/5ce8741feb2e4fdcacb968e5e50a0e83 2024-11-11T22:26:33,677 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/5ce8741feb2e4fdcacb968e5e50a0e83, entries=150, sequenceid=73, filesize=11.7 K 2024-11-11T22:26:33,678 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 467ms, sequenceid=73, compaction requested=false 2024-11-11T22:26:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:33,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-11T22:26:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-11T22:26:33,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T22:26:33,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8440 sec 2024-11-11T22:26:33,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.8490 sec 2024-11-11T22:26:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T22:26:33,983 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:33,983 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:33,984 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/7e1f5115410c4dd18797c23b6d2663ab as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/7e1f5115410c4dd18797c23b6d2663ab 2024-11-11T22:26:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-11T22:26:33,985 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:26:33,986 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:33,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:33,991 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/C of 8b5cc6a2f2e03a352bea5a20a4c912ce into 7e1f5115410c4dd18797c23b6d2663ab(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:33,991 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:33,991 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/C, priority=13, startTime=1731363993130; duration=0sec 2024-11-11T22:26:33,991 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:33,991 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:C 2024-11-11T22:26:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:26:34,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:34,117 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:34,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:26:34,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/125913d6ded54f45bc2c3e7ee362f4e9 is 50, key is test_row_0/A:col10/1731363994116/Put/seqid=0 2024-11-11T22:26:34,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742074_1250 (size=12001) 2024-11-11T22:26:34,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:26:34,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:34,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364054207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:34,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:26:34,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:34,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:26:34,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364054312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:34,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:34,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:26:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364054523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:34,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364054523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:34,524 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:34,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:34,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/125913d6ded54f45bc2c3e7ee362f4e9 2024-11-11T22:26:34,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364054533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:34,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/92b251f6b06f4112a7b2fd2258061150 is 50, key is test_row_0/B:col10/1731363994116/Put/seqid=0 2024-11-11T22:26:34,536 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:34,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:34,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364054543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:34,547 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:34,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742075_1251 (size=12001) 2024-11-11T22:26:34,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364054553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:34,556 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:34,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:34,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:26:34,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:34,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:26:34,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:26:34,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:34,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364054833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:34,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:26:34,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:34,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:34,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:34,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/92b251f6b06f4112a7b2fd2258061150 2024-11-11T22:26:34,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/fe933be0370d41df97eecfe1e46d7e02 is 50, key is test_row_0/C:col10/1731363994116/Put/seqid=0 2024-11-11T22:26:34,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742076_1252 (size=12001) 2024-11-11T22:26:35,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:26:35,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:35,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:35,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:35,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:35,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:26:35,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:26:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:35,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:35,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364055343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:35,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:26:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:35,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:35,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/fe933be0370d41df97eecfe1e46d7e02 2024-11-11T22:26:35,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/125913d6ded54f45bc2c3e7ee362f4e9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/125913d6ded54f45bc2c3e7ee362f4e9 2024-11-11T22:26:35,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/125913d6ded54f45bc2c3e7ee362f4e9, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T22:26:35,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/92b251f6b06f4112a7b2fd2258061150 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/92b251f6b06f4112a7b2fd2258061150 2024-11-11T22:26:35,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/92b251f6b06f4112a7b2fd2258061150, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T22:26:35,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/fe933be0370d41df97eecfe1e46d7e02 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/fe933be0370d41df97eecfe1e46d7e02 2024-11-11T22:26:35,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/fe933be0370d41df97eecfe1e46d7e02, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T22:26:35,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 1269ms, sequenceid=87, compaction requested=true 2024-11-11T22:26:35,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:35,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:35,386 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:35,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:35,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:35,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:35,387 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:35,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:35,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:35,388 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:35,388 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:35,388 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/A is initiating minor compaction (all files) 2024-11-11T22:26:35,388 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/B is initiating minor compaction (all files) 2024-11-11T22:26:35,388 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/A in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:35,388 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/B in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:35,388 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/15fd28e050684e3ebdb923c5709a0d06, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b9e354f6f8364d079c018cdb1f4950d9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/125913d6ded54f45bc2c3e7ee362f4e9] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=35.3 K 2024-11-11T22:26:35,388 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b8b76d8baade4adfbb7208fec7fdccf8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/43599986061a401fa3f33dec77980910, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/92b251f6b06f4112a7b2fd2258061150] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=35.3 K 2024-11-11T22:26:35,388 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15fd28e050684e3ebdb923c5709a0d06, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731363990349 2024-11-11T22:26:35,388 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting b8b76d8baade4adfbb7208fec7fdccf8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731363990349 2024-11-11T22:26:35,389 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 43599986061a401fa3f33dec77980910, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731363991932 2024-11-11T22:26:35,389 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e354f6f8364d079c018cdb1f4950d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731363991932 2024-11-11T22:26:35,389 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 92b251f6b06f4112a7b2fd2258061150, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363994109 2024-11-11T22:26:35,389 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 125913d6ded54f45bc2c3e7ee362f4e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363994109 2024-11-11T22:26:35,396 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:35,397 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/9a52af2af51d45d0bccf62133c1768ab is 50, key is test_row_0/A:col10/1731363994116/Put/seqid=0 2024-11-11T22:26:35,400 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:35,401 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/f1dbd7052f02493eb542109a7347dc83 is 50, key is test_row_0/B:col10/1731363994116/Put/seqid=0 2024-11-11T22:26:35,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742077_1253 (size=12207) 2024-11-11T22:26:35,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742078_1254 (size=12207) 2024-11-11T22:26:35,413 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/f1dbd7052f02493eb542109a7347dc83 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f1dbd7052f02493eb542109a7347dc83 2024-11-11T22:26:35,417 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/B of 8b5cc6a2f2e03a352bea5a20a4c912ce into f1dbd7052f02493eb542109a7347dc83(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:35,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:35,417 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/B, priority=13, startTime=1731363995386; duration=0sec 2024-11-11T22:26:35,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:35,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:B 2024-11-11T22:26:35,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:35,418 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:35,418 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/C is initiating minor compaction (all files) 2024-11-11T22:26:35,418 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/C in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:35,418 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/7e1f5115410c4dd18797c23b6d2663ab, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/5ce8741feb2e4fdcacb968e5e50a0e83, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/fe933be0370d41df97eecfe1e46d7e02] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=35.3 K 2024-11-11T22:26:35,419 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e1f5115410c4dd18797c23b6d2663ab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731363990349 2024-11-11T22:26:35,419 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ce8741feb2e4fdcacb968e5e50a0e83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731363991932 2024-11-11T22:26:35,419 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting fe933be0370d41df97eecfe1e46d7e02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363994109 2024-11-11T22:26:35,428 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:35,428 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/bcde66a8c4014ba7ad59c94a5b1ade7b is 50, key is test_row_0/C:col10/1731363994116/Put/seqid=0 2024-11-11T22:26:35,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742079_1255 (size=12207) 2024-11-11T22:26:35,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:26:35,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:35,516 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:26:35,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/748c0d4e23e04a8eb3513e89f9535d6b is 50, key is test_row_0/A:col10/1731363994173/Put/seqid=0 2024-11-11T22:26:35,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742080_1256 (size=12001) 2024-11-11T22:26:35,807 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/9a52af2af51d45d0bccf62133c1768ab as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9a52af2af51d45d0bccf62133c1768ab 2024-11-11T22:26:35,811 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/A of 8b5cc6a2f2e03a352bea5a20a4c912ce into 9a52af2af51d45d0bccf62133c1768ab(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:35,811 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:35,811 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/A, priority=13, startTime=1731363995386; duration=0sec 2024-11-11T22:26:35,812 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:35,812 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:A 2024-11-11T22:26:35,846 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/bcde66a8c4014ba7ad59c94a5b1ade7b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bcde66a8c4014ba7ad59c94a5b1ade7b 2024-11-11T22:26:35,851 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/C of 8b5cc6a2f2e03a352bea5a20a4c912ce into bcde66a8c4014ba7ad59c94a5b1ade7b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:35,851 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:35,851 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/C, priority=13, startTime=1731363995387; duration=0sec 2024-11-11T22:26:35,851 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:35,851 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:C 2024-11-11T22:26:35,934 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/748c0d4e23e04a8eb3513e89f9535d6b 2024-11-11T22:26:35,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/27b6df22d3ad43f183df2d4cf5047040 is 50, key is test_row_0/B:col10/1731363994173/Put/seqid=0 2024-11-11T22:26:35,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742081_1257 (size=12001) 2024-11-11T22:26:35,949 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/27b6df22d3ad43f183df2d4cf5047040 2024-11-11T22:26:35,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/8be9c73654964346a73e59268246776b is 50, key is test_row_0/C:col10/1731363994173/Put/seqid=0 2024-11-11T22:26:35,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742082_1258 (size=12001) 2024-11-11T22:26:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:26:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:36,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:36,365 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/8be9c73654964346a73e59268246776b 2024-11-11T22:26:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/748c0d4e23e04a8eb3513e89f9535d6b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/748c0d4e23e04a8eb3513e89f9535d6b 2024-11-11T22:26:36,373 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/748c0d4e23e04a8eb3513e89f9535d6b, entries=150, sequenceid=113, filesize=11.7 K 2024-11-11T22:26:36,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/27b6df22d3ad43f183df2d4cf5047040 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27b6df22d3ad43f183df2d4cf5047040 2024-11-11T22:26:36,378 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27b6df22d3ad43f183df2d4cf5047040, entries=150, sequenceid=113, filesize=11.7 K 2024-11-11T22:26:36,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364056376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:36,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/8be9c73654964346a73e59268246776b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8be9c73654964346a73e59268246776b 2024-11-11T22:26:36,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:36,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:36,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:36,382 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8be9c73654964346a73e59268246776b, entries=150, sequenceid=113, filesize=11.7 K 2024-11-11T22:26:36,383 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 867ms, sequenceid=113, compaction requested=false 2024-11-11T22:26:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-11T22:26:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-11T22:26:36,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T22:26:36,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3990 sec 2024-11-11T22:26:36,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.4040 sec 2024-11-11T22:26:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:36,485 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:36,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:26:36,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/b277b813afea43979afed6cffbb1e6f7 is 50, key is test_row_0/A:col10/1731363996374/Put/seqid=0 2024-11-11T22:26:36,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742083_1259 (size=14341) 2024-11-11T22:26:36,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364056559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:36,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364056673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364056883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:36,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/b277b813afea43979afed6cffbb1e6f7 2024-11-11T22:26:36,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/27d8d9a9348245c596b9047c9c3e55e8 is 50, key is test_row_0/B:col10/1731363996374/Put/seqid=0 2024-11-11T22:26:36,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742084_1260 (size=12001) 2024-11-11T22:26:36,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/27d8d9a9348245c596b9047c9c3e55e8 2024-11-11T22:26:36,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/035fd32916a84bd49603f8554f4c88f3 is 50, key is test_row_0/C:col10/1731363996374/Put/seqid=0 2024-11-11T22:26:36,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742085_1261 (size=12001) 2024-11-11T22:26:37,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364057193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:37,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/035fd32916a84bd49603f8554f4c88f3 2024-11-11T22:26:37,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/b277b813afea43979afed6cffbb1e6f7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b277b813afea43979afed6cffbb1e6f7 2024-11-11T22:26:37,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b277b813afea43979afed6cffbb1e6f7, entries=200, sequenceid=127, filesize=14.0 K 2024-11-11T22:26:37,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/27d8d9a9348245c596b9047c9c3e55e8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27d8d9a9348245c596b9047c9c3e55e8 2024-11-11T22:26:37,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27d8d9a9348245c596b9047c9c3e55e8, entries=150, sequenceid=127, filesize=11.7 K 2024-11-11T22:26:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/035fd32916a84bd49603f8554f4c88f3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/035fd32916a84bd49603f8554f4c88f3 2024-11-11T22:26:37,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/035fd32916a84bd49603f8554f4c88f3, entries=150, sequenceid=127, filesize=11.7 K 2024-11-11T22:26:37,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 851ms, sequenceid=127, compaction requested=true 2024-11-11T22:26:37,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:37,337 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:37,337 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:37,337 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:37,337 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:37,338 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/A is initiating minor compaction (all files) 2024-11-11T22:26:37,338 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/B is initiating minor compaction (all files) 2024-11-11T22:26:37,338 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/B in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:37,338 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f1dbd7052f02493eb542109a7347dc83, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27b6df22d3ad43f183df2d4cf5047040, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27d8d9a9348245c596b9047c9c3e55e8] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=35.4 K 2024-11-11T22:26:37,338 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/A in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:37,338 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9a52af2af51d45d0bccf62133c1768ab, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/748c0d4e23e04a8eb3513e89f9535d6b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b277b813afea43979afed6cffbb1e6f7] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=37.6 K 2024-11-11T22:26:37,338 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a52af2af51d45d0bccf62133c1768ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363994109 2024-11-11T22:26:37,338 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f1dbd7052f02493eb542109a7347dc83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363994109 2024-11-11T22:26:37,338 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b6df22d3ad43f183df2d4cf5047040, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731363994173 2024-11-11T22:26:37,339 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 748c0d4e23e04a8eb3513e89f9535d6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731363994173 2024-11-11T22:26:37,339 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 27d8d9a9348245c596b9047c9c3e55e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731363996368 2024-11-11T22:26:37,339 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b277b813afea43979afed6cffbb1e6f7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731363996368 2024-11-11T22:26:37,346 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:37,346 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/87a439e92485422fa807e4756a0269b1 is 50, key is test_row_0/A:col10/1731363996374/Put/seqid=0 2024-11-11T22:26:37,350 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:37,351 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/666bd9e5ba6d4c5a90dfad0a61eac3cd is 50, key is test_row_0/B:col10/1731363996374/Put/seqid=0 2024-11-11T22:26:37,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742086_1262 (size=12309) 2024-11-11T22:26:37,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742087_1263 (size=12309) 2024-11-11T22:26:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:37,706 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:37,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:26:37,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/9357f32e716d49f3aa3ab7d906cf4d10 is 50, key is test_row_0/A:col10/1731363996554/Put/seqid=0 2024-11-11T22:26:37,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742088_1264 (size=14541) 2024-11-11T22:26:37,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/9357f32e716d49f3aa3ab7d906cf4d10 2024-11-11T22:26:37,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/6e97f66325314231bed74b4d7df5ff24 is 50, key is test_row_0/B:col10/1731363996554/Put/seqid=0 2024-11-11T22:26:37,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742089_1265 (size=12151) 2024-11-11T22:26:37,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364057739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:37,775 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/87a439e92485422fa807e4756a0269b1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/87a439e92485422fa807e4756a0269b1 2024-11-11T22:26:37,778 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/666bd9e5ba6d4c5a90dfad0a61eac3cd as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/666bd9e5ba6d4c5a90dfad0a61eac3cd 2024-11-11T22:26:37,780 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/A of 8b5cc6a2f2e03a352bea5a20a4c912ce into 87a439e92485422fa807e4756a0269b1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:37,780 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:37,780 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/A, priority=13, startTime=1731363997336; duration=0sec 2024-11-11T22:26:37,780 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:37,780 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:A 2024-11-11T22:26:37,781 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:37,782 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:37,782 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/C is initiating minor compaction (all files) 2024-11-11T22:26:37,783 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/C in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:37,783 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bcde66a8c4014ba7ad59c94a5b1ade7b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8be9c73654964346a73e59268246776b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/035fd32916a84bd49603f8554f4c88f3] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=35.4 K 2024-11-11T22:26:37,783 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcde66a8c4014ba7ad59c94a5b1ade7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363994109 2024-11-11T22:26:37,783 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8be9c73654964346a73e59268246776b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731363994173 2024-11-11T22:26:37,784 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 035fd32916a84bd49603f8554f4c88f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731363996368 2024-11-11T22:26:37,784 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/B of 8b5cc6a2f2e03a352bea5a20a4c912ce into 666bd9e5ba6d4c5a90dfad0a61eac3cd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:37,784 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:37,784 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/B, priority=13, startTime=1731363997337; duration=0sec 2024-11-11T22:26:37,784 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:37,784 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:B 2024-11-11T22:26:37,802 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#C#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:37,803 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/31767ac4994d4597b89f9f43203dea7b is 50, key is test_row_0/C:col10/1731363996374/Put/seqid=0 2024-11-11T22:26:37,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742090_1266 (size=12309) 2024-11-11T22:26:37,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364057853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:37,919 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T22:26:37,919 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T22:26:38,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364058062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/6e97f66325314231bed74b4d7df5ff24 2024-11-11T22:26:38,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/78c5cb5033ce4583946efeb447801efd is 50, key is test_row_0/C:col10/1731363996554/Put/seqid=0 2024-11-11T22:26:38,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742091_1267 (size=12151) 2024-11-11T22:26:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:26:38,154 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:38,154 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T22:26:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:26:38,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:38,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:38,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:38,226 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/31767ac4994d4597b89f9f43203dea7b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/31767ac4994d4597b89f9f43203dea7b 2024-11-11T22:26:38,231 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/C of 8b5cc6a2f2e03a352bea5a20a4c912ce into 31767ac4994d4597b89f9f43203dea7b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:38,231 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:38,231 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/C, priority=13, startTime=1731363997337; duration=0sec 2024-11-11T22:26:38,231 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:38,232 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:C 2024-11-11T22:26:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:26:38,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:26:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:38,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364058373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:26:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:38,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:26:38,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/78c5cb5033ce4583946efeb447801efd 2024-11-11T22:26:38,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/9357f32e716d49f3aa3ab7d906cf4d10 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9357f32e716d49f3aa3ab7d906cf4d10 2024-11-11T22:26:38,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9357f32e716d49f3aa3ab7d906cf4d10, entries=200, sequenceid=152, filesize=14.2 K 2024-11-11T22:26:38,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/6e97f66325314231bed74b4d7df5ff24 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/6e97f66325314231bed74b4d7df5ff24 2024-11-11T22:26:38,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/6e97f66325314231bed74b4d7df5ff24, entries=150, sequenceid=152, filesize=11.9 K 2024-11-11T22:26:38,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/78c5cb5033ce4583946efeb447801efd as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/78c5cb5033ce4583946efeb447801efd 2024-11-11T22:26:38,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364058553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/78c5cb5033ce4583946efeb447801efd, entries=150, sequenceid=152, filesize=11.9 K 2024-11-11T22:26:38,555 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:38,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 850ms, sequenceid=152, compaction requested=false 2024-11-11T22:26:38,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:38,565 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:38,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:26:38,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/ace6cceaeb844ab4b531fa4ca6f30bb3 is 50, key is test_row_0/A:col10/1731363997733/Put/seqid=0 2024-11-11T22:26:38,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742092_1268 (size=12151) 2024-11-11T22:26:38,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/ace6cceaeb844ab4b531fa4ca6f30bb3 2024-11-11T22:26:38,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/0252767e6dae463d93742e0fe6a63fbf is 50, key is test_row_0/B:col10/1731363997733/Put/seqid=0 2024-11-11T22:26:38,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742093_1269 (size=12151) 2024-11-11T22:26:38,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364058596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364058597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364058598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:26:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:38,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364058703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364058703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364058713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:26:38,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:38,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:26:38,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364058883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364058913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364058913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:26:38,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:38,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:38,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:38,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:38,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364058923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:38,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/0252767e6dae463d93742e0fe6a63fbf 2024-11-11T22:26:38,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/99154d6732c04fc4a8aeb9df8d80ca1b is 50, key is test_row_0/C:col10/1731363997733/Put/seqid=0 2024-11-11T22:26:38,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742094_1270 (size=12151) 2024-11-11T22:26:39,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:26:39,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:39,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:39,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:39,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:39,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:39,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364059223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:39,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:26:39,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:39,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364059224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:39,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:39,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:39,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:39,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364059233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:26:39,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:26:39,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:39,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:39,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:39,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:39,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:39,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/99154d6732c04fc4a8aeb9df8d80ca1b 2024-11-11T22:26:39,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/ace6cceaeb844ab4b531fa4ca6f30bb3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ace6cceaeb844ab4b531fa4ca6f30bb3 2024-11-11T22:26:39,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ace6cceaeb844ab4b531fa4ca6f30bb3, entries=150, sequenceid=168, filesize=11.9 K 2024-11-11T22:26:39,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/0252767e6dae463d93742e0fe6a63fbf as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0252767e6dae463d93742e0fe6a63fbf 2024-11-11T22:26:39,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0252767e6dae463d93742e0fe6a63fbf, entries=150, sequenceid=168, filesize=11.9 K 2024-11-11T22:26:39,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/99154d6732c04fc4a8aeb9df8d80ca1b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/99154d6732c04fc4a8aeb9df8d80ca1b 2024-11-11T22:26:39,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/99154d6732c04fc4a8aeb9df8d80ca1b, entries=150, sequenceid=168, filesize=11.9 K 2024-11-11T22:26:39,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 849ms, sequenceid=168, compaction requested=true 2024-11-11T22:26:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:39,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:39,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:39,415 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:39,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:39,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:39,415 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:39,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:39,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:39,415 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:39,415 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:39,415 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/B is initiating minor compaction (all files) 2024-11-11T22:26:39,415 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/A is initiating minor compaction (all files) 2024-11-11T22:26:39,416 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/B in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:39,416 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/A in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:39,416 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/666bd9e5ba6d4c5a90dfad0a61eac3cd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/6e97f66325314231bed74b4d7df5ff24, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0252767e6dae463d93742e0fe6a63fbf] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=35.8 K 2024-11-11T22:26:39,416 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/87a439e92485422fa807e4756a0269b1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9357f32e716d49f3aa3ab7d906cf4d10, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ace6cceaeb844ab4b531fa4ca6f30bb3] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=38.1 K 2024-11-11T22:26:39,416 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 666bd9e5ba6d4c5a90dfad0a61eac3cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731363996368 2024-11-11T22:26:39,416 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87a439e92485422fa807e4756a0269b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731363996368 2024-11-11T22:26:39,416 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9357f32e716d49f3aa3ab7d906cf4d10, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731363996549 2024-11-11T22:26:39,416 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e97f66325314231bed74b4d7df5ff24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731363996549 2024-11-11T22:26:39,416 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting ace6cceaeb844ab4b531fa4ca6f30bb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731363997733 2024-11-11T22:26:39,416 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 0252767e6dae463d93742e0fe6a63fbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731363997733 2024-11-11T22:26:39,423 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:39,423 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#A#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:39,424 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/5ec090771fe946efb5d08cc888e640de is 50, key is test_row_0/B:col10/1731363997733/Put/seqid=0 2024-11-11T22:26:39,424 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/2dc4a68677a74f10b7ce6da93dcff92a is 50, key is test_row_0/A:col10/1731363997733/Put/seqid=0 2024-11-11T22:26:39,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742096_1272 (size=12561) 2024-11-11T22:26:39,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742095_1271 (size=12561) 2024-11-11T22:26:39,434 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/2dc4a68677a74f10b7ce6da93dcff92a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/2dc4a68677a74f10b7ce6da93dcff92a 2024-11-11T22:26:39,438 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/A of 8b5cc6a2f2e03a352bea5a20a4c912ce into 2dc4a68677a74f10b7ce6da93dcff92a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:39,438 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:39,438 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/A, priority=13, startTime=1731363999414; duration=0sec 2024-11-11T22:26:39,438 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:39,438 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:A 2024-11-11T22:26:39,438 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:39,439 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:39,439 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/C is initiating minor compaction (all files) 2024-11-11T22:26:39,439 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/C in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:39,439 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/31767ac4994d4597b89f9f43203dea7b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/78c5cb5033ce4583946efeb447801efd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/99154d6732c04fc4a8aeb9df8d80ca1b] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=35.8 K 2024-11-11T22:26:39,439 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31767ac4994d4597b89f9f43203dea7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731363996368 2024-11-11T22:26:39,440 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78c5cb5033ce4583946efeb447801efd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731363996549 2024-11-11T22:26:39,440 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99154d6732c04fc4a8aeb9df8d80ca1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731363997733 2024-11-11T22:26:39,447 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:39,448 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/75b9f5ab76db42859a1f9bf8a7666989 is 50, key is test_row_0/C:col10/1731363997733/Put/seqid=0 2024-11-11T22:26:39,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742097_1273 (size=12561) 2024-11-11T22:26:39,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:26:39,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:39,530 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:26:39,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/be708e90410349f1b0e9117d0f12ee7e is 50, key is test_row_0/A:col10/1731363998592/Put/seqid=0 2024-11-11T22:26:39,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742098_1274 (size=12151) 2024-11-11T22:26:39,540 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/be708e90410349f1b0e9117d0f12ee7e 2024-11-11T22:26:39,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/0db792651abb40d3a62ff071269d1937 is 50, key is test_row_0/B:col10/1731363998592/Put/seqid=0 2024-11-11T22:26:39,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742099_1275 (size=12151) 2024-11-11T22:26:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:39,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:39,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364059746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364059746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364059748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:39,833 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/5ec090771fe946efb5d08cc888e640de as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/5ec090771fe946efb5d08cc888e640de 2024-11-11T22:26:39,841 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/B of 8b5cc6a2f2e03a352bea5a20a4c912ce into 5ec090771fe946efb5d08cc888e640de(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:39,841 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:39,841 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/B, priority=13, startTime=1731363999415; duration=0sec 2024-11-11T22:26:39,841 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:39,841 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:B 2024-11-11T22:26:39,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364059853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:39,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364059853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:39,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364059853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:39,857 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/75b9f5ab76db42859a1f9bf8a7666989 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/75b9f5ab76db42859a1f9bf8a7666989 2024-11-11T22:26:39,861 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/C of 8b5cc6a2f2e03a352bea5a20a4c912ce into 75b9f5ab76db42859a1f9bf8a7666989(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:39,861 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:39,861 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/C, priority=13, startTime=1731363999415; duration=0sec 2024-11-11T22:26:39,861 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:39,861 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:C 2024-11-11T22:26:39,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364059892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:39,894 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:39,952 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/0db792651abb40d3a62ff071269d1937 2024-11-11T22:26:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d3d6e3f7ebfe454bae7d818cf7fb5bc7 is 50, key is test_row_0/C:col10/1731363998592/Put/seqid=0 2024-11-11T22:26:39,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742100_1276 (size=12151) 2024-11-11T22:26:40,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364060063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:40,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364060063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:40,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364060063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:26:40,363 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d3d6e3f7ebfe454bae7d818cf7fb5bc7 2024-11-11T22:26:40,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/be708e90410349f1b0e9117d0f12ee7e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be708e90410349f1b0e9117d0f12ee7e 2024-11-11T22:26:40,371 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be708e90410349f1b0e9117d0f12ee7e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T22:26:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/0db792651abb40d3a62ff071269d1937 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0db792651abb40d3a62ff071269d1937 2024-11-11T22:26:40,375 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0db792651abb40d3a62ff071269d1937, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T22:26:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364060373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:40,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d3d6e3f7ebfe454bae7d818cf7fb5bc7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d3d6e3f7ebfe454bae7d818cf7fb5bc7 2024-11-11T22:26:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364060374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364060374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:40,380 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d3d6e3f7ebfe454bae7d818cf7fb5bc7, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T22:26:40,381 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 851ms, sequenceid=193, compaction requested=false 2024-11-11T22:26:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T22:26:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T22:26:40,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T22:26:40,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2260 sec 2024-11-11T22:26:40,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.2310 sec 2024-11-11T22:26:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:40,886 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:40,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:26:40,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/513e7adde3e248e58a12c437bd3a1666 is 50, key is test_row_0/A:col10/1731363999747/Put/seqid=0 2024-11-11T22:26:40,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742101_1277 (size=12151) 2024-11-11T22:26:40,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:40,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364060915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364060916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:40,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364060916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364061023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:41,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364061023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364061023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364061233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364061234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:41,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364061234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/513e7adde3e248e58a12c437bd3a1666 2024-11-11T22:26:41,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/0a12d02129a94da1b78d539d99147431 is 50, key is test_row_0/B:col10/1731363999747/Put/seqid=0 2024-11-11T22:26:41,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742102_1278 (size=12151) 2024-11-11T22:26:41,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364061543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:41,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:41,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364061543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:41,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364061543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:41,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/0a12d02129a94da1b78d539d99147431 2024-11-11T22:26:41,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/213cb5c3ccdb41e1a1c460caacd3e5ae is 50, key is test_row_0/C:col10/1731363999747/Put/seqid=0 2024-11-11T22:26:41,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742103_1279 (size=12151) 2024-11-11T22:26:41,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/213cb5c3ccdb41e1a1c460caacd3e5ae 2024-11-11T22:26:41,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/513e7adde3e248e58a12c437bd3a1666 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/513e7adde3e248e58a12c437bd3a1666 2024-11-11T22:26:41,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/513e7adde3e248e58a12c437bd3a1666, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T22:26:41,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/0a12d02129a94da1b78d539d99147431 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0a12d02129a94da1b78d539d99147431 2024-11-11T22:26:41,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0a12d02129a94da1b78d539d99147431, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T22:26:41,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/213cb5c3ccdb41e1a1c460caacd3e5ae as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/213cb5c3ccdb41e1a1c460caacd3e5ae 2024-11-11T22:26:41,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/213cb5c3ccdb41e1a1c460caacd3e5ae, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T22:26:41,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 881ms, sequenceid=209, compaction requested=true 2024-11-11T22:26:41,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:41,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:41,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:41,767 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:41,767 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:41,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:41,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:41,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:41,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:41,768 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:41,769 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/A is initiating minor compaction (all files) 2024-11-11T22:26:41,769 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/A in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:41,769 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/2dc4a68677a74f10b7ce6da93dcff92a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be708e90410349f1b0e9117d0f12ee7e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/513e7adde3e248e58a12c437bd3a1666] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=36.0 K 2024-11-11T22:26:41,769 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:41,769 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/B is initiating minor compaction (all files) 2024-11-11T22:26:41,769 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/B in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:41,769 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/5ec090771fe946efb5d08cc888e640de, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0db792651abb40d3a62ff071269d1937, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0a12d02129a94da1b78d539d99147431] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=36.0 K 2024-11-11T22:26:41,769 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dc4a68677a74f10b7ce6da93dcff92a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731363997733 2024-11-11T22:26:41,770 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec090771fe946efb5d08cc888e640de, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731363997733 2024-11-11T22:26:41,770 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting be708e90410349f1b0e9117d0f12ee7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731363998580 2024-11-11T22:26:41,771 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db792651abb40d3a62ff071269d1937, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731363998580 2024-11-11T22:26:41,771 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 513e7adde3e248e58a12c437bd3a1666, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731363999737 2024-11-11T22:26:41,771 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a12d02129a94da1b78d539d99147431, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731363999737 2024-11-11T22:26:41,787 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:41,787 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:41,787 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/10e6de13d6494ce88cf4618100797c59 is 50, key is test_row_0/A:col10/1731363999747/Put/seqid=0 2024-11-11T22:26:41,787 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/36b98daea5f447629bddaa108f575739 is 50, key is test_row_0/B:col10/1731363999747/Put/seqid=0 2024-11-11T22:26:41,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742104_1280 (size=12663) 2024-11-11T22:26:41,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742105_1281 (size=12663) 2024-11-11T22:26:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:41,907 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:41,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:26:41,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/d41deac6d468479f88dcbc8f5bd3817e is 50, key is test_row_0/A:col10/1731364001906/Put/seqid=0 2024-11-11T22:26:41,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742106_1282 (size=14541) 2024-11-11T22:26:41,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/d41deac6d468479f88dcbc8f5bd3817e 2024-11-11T22:26:41,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/575c5fce6a1e46aa9498fc3ff3843c19 is 50, key is test_row_0/B:col10/1731364001906/Put/seqid=0 2024-11-11T22:26:41,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364061958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:41,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742107_1283 (size=12151) 2024-11-11T22:26:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364062054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364062054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364062054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:42,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364062073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:42,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:42,217 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/36b98daea5f447629bddaa108f575739 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/36b98daea5f447629bddaa108f575739 2024-11-11T22:26:42,223 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/B of 8b5cc6a2f2e03a352bea5a20a4c912ce into 36b98daea5f447629bddaa108f575739(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:42,224 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:42,224 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/B, priority=13, startTime=1731364001767; duration=0sec 2024-11-11T22:26:42,224 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:42,224 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:B 2024-11-11T22:26:42,224 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:42,225 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:42,226 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/C is initiating minor compaction (all files) 2024-11-11T22:26:42,226 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/C in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:42,226 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/75b9f5ab76db42859a1f9bf8a7666989, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d3d6e3f7ebfe454bae7d818cf7fb5bc7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/213cb5c3ccdb41e1a1c460caacd3e5ae] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=36.0 K 2024-11-11T22:26:42,227 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b9f5ab76db42859a1f9bf8a7666989, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731363997733 2024-11-11T22:26:42,227 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d6e3f7ebfe454bae7d818cf7fb5bc7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731363998580 2024-11-11T22:26:42,227 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 213cb5c3ccdb41e1a1c460caacd3e5ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731363999737 2024-11-11T22:26:42,233 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/10e6de13d6494ce88cf4618100797c59 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/10e6de13d6494ce88cf4618100797c59 2024-11-11T22:26:42,239 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#C#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:42,240 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/45e839a83c6442cfb71bbcaec444cdfc is 50, key is test_row_0/C:col10/1731363999747/Put/seqid=0 2024-11-11T22:26:42,241 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/A of 8b5cc6a2f2e03a352bea5a20a4c912ce into 10e6de13d6494ce88cf4618100797c59(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:42,241 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:42,241 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/A, priority=13, startTime=1731364001767; duration=0sec 2024-11-11T22:26:42,241 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:42,242 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:A 2024-11-11T22:26:42,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742108_1284 (size=12663) 2024-11-11T22:26:42,263 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/45e839a83c6442cfb71bbcaec444cdfc as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/45e839a83c6442cfb71bbcaec444cdfc 2024-11-11T22:26:42,268 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/C of 8b5cc6a2f2e03a352bea5a20a4c912ce into 45e839a83c6442cfb71bbcaec444cdfc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:42,268 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:42,268 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/C, priority=13, startTime=1731364001768; duration=0sec 2024-11-11T22:26:42,269 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:42,269 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:C 2024-11-11T22:26:42,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:42,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364062283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:26:42,333 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:42,334 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T22:26:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:26:42,338 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:42,339 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:42,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:42,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/575c5fce6a1e46aa9498fc3ff3843c19 2024-11-11T22:26:42,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/918b53b4f78a43e3b6dc3a2c8d9b529f is 50, key is test_row_0/C:col10/1731364001906/Put/seqid=0 2024-11-11T22:26:42,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742109_1285 (size=12151) 2024-11-11T22:26:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:26:42,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T22:26:42,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:42,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:42,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:42,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:42,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364062593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:42,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T22:26:42,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:42,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:42,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:42,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:42,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:26:42,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/918b53b4f78a43e3b6dc3a2c8d9b529f 2024-11-11T22:26:42,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/d41deac6d468479f88dcbc8f5bd3817e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/d41deac6d468479f88dcbc8f5bd3817e 2024-11-11T22:26:42,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/d41deac6d468479f88dcbc8f5bd3817e, entries=200, sequenceid=232, filesize=14.2 K 2024-11-11T22:26:42,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/575c5fce6a1e46aa9498fc3ff3843c19 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/575c5fce6a1e46aa9498fc3ff3843c19 2024-11-11T22:26:42,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/575c5fce6a1e46aa9498fc3ff3843c19, entries=150, sequenceid=232, filesize=11.9 K 2024-11-11T22:26:42,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/918b53b4f78a43e3b6dc3a2c8d9b529f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/918b53b4f78a43e3b6dc3a2c8d9b529f 2024-11-11T22:26:42,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T22:26:42,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:42,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:42,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:42,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:42,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:42,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/918b53b4f78a43e3b6dc3a2c8d9b529f, entries=150, sequenceid=232, filesize=11.9 K 2024-11-11T22:26:42,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 898ms, sequenceid=232, compaction requested=false 2024-11-11T22:26:42,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:42,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T22:26:42,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:42,952 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T22:26:42,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/ed4741f71bcf43e190aa9bdaff71ccb9 is 50, key is test_row_0/A:col10/1731364001956/Put/seqid=0 2024-11-11T22:26:42,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742110_1286 (size=12151) 2024-11-11T22:26:42,963 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/ed4741f71bcf43e190aa9bdaff71ccb9 2024-11-11T22:26:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:26:42,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/4f2e7401c40443a08579f1ff478c1417 is 50, key is test_row_0/B:col10/1731364001956/Put/seqid=0 2024-11-11T22:26:42,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742111_1287 (size=12151) 2024-11-11T22:26:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:43,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:43,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364063099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364063100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364063102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364063103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364063214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364063215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364063215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,377 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/4f2e7401c40443a08579f1ff478c1417 2024-11-11T22:26:43,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/cf49cce69d9d4a3cbaf34f42a7705acb is 50, key is test_row_0/C:col10/1731364001956/Put/seqid=0 2024-11-11T22:26:43,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742112_1288 (size=12151) 2024-11-11T22:26:43,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364063424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364063424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364063424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:26:43,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364063733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364063733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364063733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:43,827 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/cf49cce69d9d4a3cbaf34f42a7705acb 2024-11-11T22:26:43,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/ed4741f71bcf43e190aa9bdaff71ccb9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ed4741f71bcf43e190aa9bdaff71ccb9 2024-11-11T22:26:43,835 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ed4741f71bcf43e190aa9bdaff71ccb9, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T22:26:43,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/4f2e7401c40443a08579f1ff478c1417 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f2e7401c40443a08579f1ff478c1417 2024-11-11T22:26:43,841 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f2e7401c40443a08579f1ff478c1417, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T22:26:43,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/cf49cce69d9d4a3cbaf34f42a7705acb as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/cf49cce69d9d4a3cbaf34f42a7705acb 2024-11-11T22:26:43,848 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/cf49cce69d9d4a3cbaf34f42a7705acb, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T22:26:43,848 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 896ms, sequenceid=248, compaction requested=true 2024-11-11T22:26:43,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:43,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:43,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T22:26:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T22:26:43,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T22:26:43,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5110 sec 2024-11-11T22:26:43,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5190 sec 2024-11-11T22:26:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:44,116 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:44,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:26:44,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/28a82fedd8ca4bada70cfe3f1d1dfd0f is 50, key is test_row_0/A:col10/1731364003100/Put/seqid=0 2024-11-11T22:26:44,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742113_1289 (size=12301) 2024-11-11T22:26:44,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/28a82fedd8ca4bada70cfe3f1d1dfd0f 2024-11-11T22:26:44,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/475fd4b28f6449a5bb0003d6a22408bf is 50, key is test_row_0/B:col10/1731364003100/Put/seqid=0 2024-11-11T22:26:44,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742114_1290 (size=12301) 2024-11-11T22:26:44,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364064158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:44,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364064244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364064244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:44,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364064253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:44,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364064273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:26:44,483 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:44,484 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:44,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364064483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T22:26:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:26:44,487 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:44,488 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:44,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:44,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/475fd4b28f6449a5bb0003d6a22408bf 2024-11-11T22:26:44,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d2df0486a082410fa2b84cc9940afc06 is 50, key is test_row_0/C:col10/1731364003100/Put/seqid=0 2024-11-11T22:26:44,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742115_1291 (size=12301) 2024-11-11T22:26:44,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d2df0486a082410fa2b84cc9940afc06 2024-11-11T22:26:44,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/28a82fedd8ca4bada70cfe3f1d1dfd0f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/28a82fedd8ca4bada70cfe3f1d1dfd0f 2024-11-11T22:26:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:26:44,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/28a82fedd8ca4bada70cfe3f1d1dfd0f, entries=150, sequenceid=272, filesize=12.0 K 2024-11-11T22:26:44,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/475fd4b28f6449a5bb0003d6a22408bf as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/475fd4b28f6449a5bb0003d6a22408bf 2024-11-11T22:26:44,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/475fd4b28f6449a5bb0003d6a22408bf, entries=150, sequenceid=272, filesize=12.0 K 2024-11-11T22:26:44,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d2df0486a082410fa2b84cc9940afc06 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d2df0486a082410fa2b84cc9940afc06 2024-11-11T22:26:44,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d2df0486a082410fa2b84cc9940afc06, entries=150, sequenceid=272, filesize=12.0 K 2024-11-11T22:26:44,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 496ms, sequenceid=272, compaction requested=true 2024-11-11T22:26:44,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:44,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:44,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:44,613 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:44,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:44,613 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:44,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:44,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:44,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:44,615 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51656 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:44,615 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:44,615 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/B is initiating minor compaction (all files) 2024-11-11T22:26:44,615 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/A is initiating minor compaction (all files) 2024-11-11T22:26:44,615 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/A in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:44,615 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/B in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:44,615 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/36b98daea5f447629bddaa108f575739, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/575c5fce6a1e46aa9498fc3ff3843c19, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f2e7401c40443a08579f1ff478c1417, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/475fd4b28f6449a5bb0003d6a22408bf] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=48.1 K 2024-11-11T22:26:44,615 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/10e6de13d6494ce88cf4618100797c59, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/d41deac6d468479f88dcbc8f5bd3817e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ed4741f71bcf43e190aa9bdaff71ccb9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/28a82fedd8ca4bada70cfe3f1d1dfd0f] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=50.4 K 2024-11-11T22:26:44,615 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10e6de13d6494ce88cf4618100797c59, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731363999737 2024-11-11T22:26:44,615 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b98daea5f447629bddaa108f575739, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731363999737 2024-11-11T22:26:44,616 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting d41deac6d468479f88dcbc8f5bd3817e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731364000908 2024-11-11T22:26:44,616 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 575c5fce6a1e46aa9498fc3ff3843c19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731364000908 2024-11-11T22:26:44,616 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f2e7401c40443a08579f1ff478c1417, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731364001924 2024-11-11T22:26:44,616 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed4741f71bcf43e190aa9bdaff71ccb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731364001924 2024-11-11T22:26:44,616 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a82fedd8ca4bada70cfe3f1d1dfd0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731364003089 2024-11-11T22:26:44,616 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 475fd4b28f6449a5bb0003d6a22408bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731364003089 2024-11-11T22:26:44,640 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:44,641 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/5a57ed8313ef45ffae4ee79ad088d547 is 50, key is test_row_0/A:col10/1731364003100/Put/seqid=0 2024-11-11T22:26:44,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T22:26:44,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:44,644 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:26:44,650 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:44,651 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/68b733684b24493ca1ca0c6a76de44ee is 50, key is test_row_0/B:col10/1731364003100/Put/seqid=0 2024-11-11T22:26:44,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/00b78e9e169f445b881f5b4caa09421f is 50, key is test_row_0/A:col10/1731364004156/Put/seqid=0 2024-11-11T22:26:44,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742116_1292 (size=12949) 2024-11-11T22:26:44,718 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/5a57ed8313ef45ffae4ee79ad088d547 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5a57ed8313ef45ffae4ee79ad088d547 2024-11-11T22:26:44,726 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/A of 8b5cc6a2f2e03a352bea5a20a4c912ce into 5a57ed8313ef45ffae4ee79ad088d547(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:44,726 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:44,727 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/A, priority=12, startTime=1731364004613; duration=0sec 2024-11-11T22:26:44,727 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:44,727 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:A 2024-11-11T22:26:44,727 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:44,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742118_1294 (size=12301) 2024-11-11T22:26:44,730 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/00b78e9e169f445b881f5b4caa09421f 2024-11-11T22:26:44,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742117_1293 (size=12949) 2024-11-11T22:26:44,736 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:44,736 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/C is initiating minor compaction (all files) 2024-11-11T22:26:44,736 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/C in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:44,736 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/45e839a83c6442cfb71bbcaec444cdfc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/918b53b4f78a43e3b6dc3a2c8d9b529f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/cf49cce69d9d4a3cbaf34f42a7705acb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d2df0486a082410fa2b84cc9940afc06] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=48.1 K 2024-11-11T22:26:44,738 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45e839a83c6442cfb71bbcaec444cdfc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731363999737 2024-11-11T22:26:44,739 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918b53b4f78a43e3b6dc3a2c8d9b529f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731364000908 2024-11-11T22:26:44,739 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf49cce69d9d4a3cbaf34f42a7705acb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731364001924 2024-11-11T22:26:44,740 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2df0486a082410fa2b84cc9940afc06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731364003089 2024-11-11T22:26:44,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/9e1a4469c2004f658e55f4aebf4cce3c is 50, key is test_row_0/B:col10/1731364004156/Put/seqid=0 2024-11-11T22:26:44,779 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#C#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:44,779 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/8f73bba6d2d349adb2a28478f51f5879 is 50, key is test_row_0/C:col10/1731364003100/Put/seqid=0 2024-11-11T22:26:44,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:44,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742119_1295 (size=12301) 2024-11-11T22:26:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:26:44,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742120_1296 (size=12949) 2024-11-11T22:26:44,829 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/8f73bba6d2d349adb2a28478f51f5879 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8f73bba6d2d349adb2a28478f51f5879 2024-11-11T22:26:44,836 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/C of 8b5cc6a2f2e03a352bea5a20a4c912ce into 8f73bba6d2d349adb2a28478f51f5879(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:44,836 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:44,836 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/C, priority=12, startTime=1731364004613; duration=0sec 2024-11-11T22:26:44,836 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:44,836 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:C 2024-11-11T22:26:44,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364064897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364065003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:26:45,141 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/68b733684b24493ca1ca0c6a76de44ee as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/68b733684b24493ca1ca0c6a76de44ee 2024-11-11T22:26:45,149 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/B of 8b5cc6a2f2e03a352bea5a20a4c912ce into 68b733684b24493ca1ca0c6a76de44ee(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:45,149 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:45,149 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/B, priority=12, startTime=1731364004613; duration=0sec 2024-11-11T22:26:45,150 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:45,150 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:B 2024-11-11T22:26:45,197 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/9e1a4469c2004f658e55f4aebf4cce3c 2024-11-11T22:26:45,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/563118c861cb4bcaaff69e858a6d8412 is 50, key is test_row_0/C:col10/1731364004156/Put/seqid=0 2024-11-11T22:26:45,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364065213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742121_1297 (size=12301) 2024-11-11T22:26:45,220 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/563118c861cb4bcaaff69e858a6d8412 2024-11-11T22:26:45,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/00b78e9e169f445b881f5b4caa09421f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/00b78e9e169f445b881f5b4caa09421f 2024-11-11T22:26:45,231 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/00b78e9e169f445b881f5b4caa09421f, entries=150, sequenceid=284, filesize=12.0 K 2024-11-11T22:26:45,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/9e1a4469c2004f658e55f4aebf4cce3c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/9e1a4469c2004f658e55f4aebf4cce3c 2024-11-11T22:26:45,242 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/9e1a4469c2004f658e55f4aebf4cce3c, entries=150, sequenceid=284, filesize=12.0 K 2024-11-11T22:26:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/563118c861cb4bcaaff69e858a6d8412 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/563118c861cb4bcaaff69e858a6d8412 2024-11-11T22:26:45,250 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/563118c861cb4bcaaff69e858a6d8412, entries=150, sequenceid=284, filesize=12.0 K 2024-11-11T22:26:45,252 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 608ms, sequenceid=284, compaction requested=false 2024-11-11T22:26:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T22:26:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T22:26:45,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T22:26:45,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 766 msec 2024-11-11T22:26:45,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 774 msec 2024-11-11T22:26:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:45,268 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:45,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:26:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364065282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/b511188062294df8bef69b579843ad44 is 50, key is test_row_0/A:col10/1731364004896/Put/seqid=0 2024-11-11T22:26:45,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364065289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364065289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742122_1298 (size=14741) 2024-11-11T22:26:45,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/b511188062294df8bef69b579843ad44 2024-11-11T22:26:45,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/a6c3e495515a4a659ac02e20a129b659 is 50, key is test_row_0/B:col10/1731364004896/Put/seqid=0 2024-11-11T22:26:45,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742123_1299 (size=12301) 2024-11-11T22:26:45,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/a6c3e495515a4a659ac02e20a129b659 2024-11-11T22:26:45,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d04fa4fba7934016ba40448648f81c45 is 50, key is test_row_0/C:col10/1731364004896/Put/seqid=0 2024-11-11T22:26:45,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742124_1300 (size=12301) 2024-11-11T22:26:45,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d04fa4fba7934016ba40448648f81c45 2024-11-11T22:26:45,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/b511188062294df8bef69b579843ad44 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b511188062294df8bef69b579843ad44 2024-11-11T22:26:45,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b511188062294df8bef69b579843ad44, entries=200, sequenceid=313, filesize=14.4 K 2024-11-11T22:26:45,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/a6c3e495515a4a659ac02e20a129b659 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/a6c3e495515a4a659ac02e20a129b659 2024-11-11T22:26:45,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364065393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364065393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364065393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/a6c3e495515a4a659ac02e20a129b659, entries=150, sequenceid=313, filesize=12.0 K 2024-11-11T22:26:45,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d04fa4fba7934016ba40448648f81c45 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d04fa4fba7934016ba40448648f81c45 2024-11-11T22:26:45,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d04fa4fba7934016ba40448648f81c45, entries=150, sequenceid=313, filesize=12.0 K 2024-11-11T22:26:45,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 146ms, sequenceid=313, compaction requested=true 2024-11-11T22:26:45,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:45,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:45,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:45,414 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:45,414 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:45,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:45,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:45,417 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:45,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:45,417 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/A is initiating minor compaction (all files) 2024-11-11T22:26:45,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/B is initiating minor compaction (all files) 2024-11-11T22:26:45,417 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/A in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:45,417 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/B in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:45,417 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/68b733684b24493ca1ca0c6a76de44ee, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/9e1a4469c2004f658e55f4aebf4cce3c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/a6c3e495515a4a659ac02e20a129b659] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=36.7 K 2024-11-11T22:26:45,417 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5a57ed8313ef45ffae4ee79ad088d547, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/00b78e9e169f445b881f5b4caa09421f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b511188062294df8bef69b579843ad44] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=39.1 K 2024-11-11T22:26:45,417 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a57ed8313ef45ffae4ee79ad088d547, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731364003089 2024-11-11T22:26:45,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 68b733684b24493ca1ca0c6a76de44ee, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731364003089 2024-11-11T22:26:45,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:45,418 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e1a4469c2004f658e55f4aebf4cce3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731364004136 2024-11-11T22:26:45,418 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00b78e9e169f445b881f5b4caa09421f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731364004136 2024-11-11T22:26:45,418 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b511188062294df8bef69b579843ad44, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731364004870 2024-11-11T22:26:45,418 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c3e495515a4a659ac02e20a129b659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731364004870 2024-11-11T22:26:45,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:45,438 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:45,439 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:45,439 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/dbac5035a5204f66b264eadc81d91d3f is 50, key is test_row_0/A:col10/1731364004896/Put/seqid=0 2024-11-11T22:26:45,439 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/464b5c444f104616ab5adc85088f85fa is 50, key is test_row_0/B:col10/1731364004896/Put/seqid=0 2024-11-11T22:26:45,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742125_1301 (size=13051) 2024-11-11T22:26:45,480 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/dbac5035a5204f66b264eadc81d91d3f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/dbac5035a5204f66b264eadc81d91d3f 2024-11-11T22:26:45,486 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/A of 8b5cc6a2f2e03a352bea5a20a4c912ce into dbac5035a5204f66b264eadc81d91d3f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:45,486 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:45,486 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/A, priority=13, startTime=1731364005414; duration=0sec 2024-11-11T22:26:45,486 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:45,486 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:A 2024-11-11T22:26:45,487 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:45,488 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:45,488 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/C is initiating minor compaction (all files) 2024-11-11T22:26:45,488 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/C in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:45,488 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8f73bba6d2d349adb2a28478f51f5879, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/563118c861cb4bcaaff69e858a6d8412, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d04fa4fba7934016ba40448648f81c45] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=36.7 K 2024-11-11T22:26:45,488 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f73bba6d2d349adb2a28478f51f5879, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731364003089 2024-11-11T22:26:45,489 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 563118c861cb4bcaaff69e858a6d8412, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731364004136 2024-11-11T22:26:45,489 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting d04fa4fba7934016ba40448648f81c45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731364004870 2024-11-11T22:26:45,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742126_1302 (size=13051) 2024-11-11T22:26:45,516 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#C#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:45,516 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/7e886a378074411f90e89d9f2162f9b3 is 50, key is test_row_0/C:col10/1731364004896/Put/seqid=0 2024-11-11T22:26:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:45,526 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:45,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:26:45,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/396bf78d78ac49ae9b03cba17f662626 is 50, key is test_row_0/A:col10/1731364005272/Put/seqid=0 2024-11-11T22:26:45,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742127_1303 (size=13051) 2024-11-11T22:26:45,573 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/7e886a378074411f90e89d9f2162f9b3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/7e886a378074411f90e89d9f2162f9b3 2024-11-11T22:26:45,581 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/C of 8b5cc6a2f2e03a352bea5a20a4c912ce into 7e886a378074411f90e89d9f2162f9b3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:45,582 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:45,582 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/C, priority=13, startTime=1731364005415; duration=0sec 2024-11-11T22:26:45,582 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:45,582 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:C 2024-11-11T22:26:45,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742128_1304 (size=14741) 2024-11-11T22:26:45,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/396bf78d78ac49ae9b03cba17f662626 2024-11-11T22:26:45,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/1be5000d16544e5aa4bce4205771e2c7 is 50, key is test_row_0/B:col10/1731364005272/Put/seqid=0 2024-11-11T22:26:45,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364065613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364065616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:26:45,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,624 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364065618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,625 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T22:26:45,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364065622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:26:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,627 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:45,628 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:45,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:45,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742129_1305 (size=12301) 2024-11-11T22:26:45,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364065723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364065723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:26:45,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364065733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364065734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T22:26:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:45,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:45,917 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/464b5c444f104616ab5adc85088f85fa as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/464b5c444f104616ab5adc85088f85fa 2024-11-11T22:26:45,925 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/B of 8b5cc6a2f2e03a352bea5a20a4c912ce into 464b5c444f104616ab5adc85088f85fa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:45,925 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:45,925 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/B, priority=13, startTime=1731364005414; duration=0sec 2024-11-11T22:26:45,925 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:45,925 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:B 2024-11-11T22:26:45,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364065933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364065933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T22:26:45,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:45,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:45,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:45,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:45,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:26:45,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364065944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364065944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,032 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:26:46,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/1be5000d16544e5aa4bce4205771e2c7 2024-11-11T22:26:46,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/0b87ee0cd1b04acc98ebb0d9ad730f60 is 50, key is test_row_0/C:col10/1731364005272/Put/seqid=0 2024-11-11T22:26:46,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742130_1306 (size=12301) 2024-11-11T22:26:46,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/0b87ee0cd1b04acc98ebb0d9ad730f60 2024-11-11T22:26:46,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/396bf78d78ac49ae9b03cba17f662626 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/396bf78d78ac49ae9b03cba17f662626 2024-11-11T22:26:46,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/396bf78d78ac49ae9b03cba17f662626, entries=200, sequenceid=326, filesize=14.4 K 2024-11-11T22:26:46,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/1be5000d16544e5aa4bce4205771e2c7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/1be5000d16544e5aa4bce4205771e2c7 2024-11-11T22:26:46,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/1be5000d16544e5aa4bce4205771e2c7, entries=150, sequenceid=326, filesize=12.0 K 2024-11-11T22:26:46,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/0b87ee0cd1b04acc98ebb0d9ad730f60 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/0b87ee0cd1b04acc98ebb0d9ad730f60 2024-11-11T22:26:46,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T22:26:46,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:46,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:46,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:46,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:46,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:46,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/0b87ee0cd1b04acc98ebb0d9ad730f60, entries=150, sequenceid=326, filesize=12.0 K 2024-11-11T22:26:46,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 573ms, sequenceid=326, compaction requested=false 2024-11-11T22:26:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:46,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T22:26:46,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:46,242 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:26:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:46,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/36cb951050b14525ac130d2939153355 is 50, key is test_row_0/A:col10/1731364005613/Put/seqid=0 2024-11-11T22:26:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:26:46,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364066257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364066258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364066261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742131_1307 (size=12301) 2024-11-11T22:26:46,266 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/36cb951050b14525ac130d2939153355 2024-11-11T22:26:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364066263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/b9aa03f1baef468baeac6aa708176614 is 50, key is test_row_0/B:col10/1731364005613/Put/seqid=0 2024-11-11T22:26:46,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742132_1308 (size=12301) 2024-11-11T22:26:46,324 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/b9aa03f1baef468baeac6aa708176614 2024-11-11T22:26:46,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/204cfe04159147938f5dbaad4f96aabf is 50, key is test_row_0/C:col10/1731364005613/Put/seqid=0 2024-11-11T22:26:46,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742133_1309 (size=12301) 2024-11-11T22:26:46,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364066374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364066374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364066374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364066374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364066583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364066583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364066583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364066583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,760 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/204cfe04159147938f5dbaad4f96aabf 2024-11-11T22:26:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:26:46,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/36cb951050b14525ac130d2939153355 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/36cb951050b14525ac130d2939153355 2024-11-11T22:26:46,769 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/36cb951050b14525ac130d2939153355, entries=150, sequenceid=352, filesize=12.0 K 2024-11-11T22:26:46,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/b9aa03f1baef468baeac6aa708176614 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b9aa03f1baef468baeac6aa708176614 2024-11-11T22:26:46,774 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b9aa03f1baef468baeac6aa708176614, entries=150, sequenceid=352, filesize=12.0 K 2024-11-11T22:26:46,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/204cfe04159147938f5dbaad4f96aabf as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/204cfe04159147938f5dbaad4f96aabf 2024-11-11T22:26:46,779 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/204cfe04159147938f5dbaad4f96aabf, entries=150, sequenceid=352, filesize=12.0 K 2024-11-11T22:26:46,779 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 537ms, sequenceid=352, compaction requested=true 2024-11-11T22:26:46,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:46,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:46,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T22:26:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T22:26:46,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T22:26:46,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1530 sec 2024-11-11T22:26:46,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.1580 sec 2024-11-11T22:26:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:46,895 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:46,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T22:26:46,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/be079ede7c914204aa0f99ad82e9e24e is 50, key is test_row_0/A:col10/1731364006251/Put/seqid=0 2024-11-11T22:26:46,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742134_1310 (size=12301) 2024-11-11T22:26:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364066924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364066925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364066926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364066927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:46,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:46,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364067033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364067043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364067043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364067043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364067243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364067252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364067252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364067253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/be079ede7c914204aa0f99ad82e9e24e 2024-11-11T22:26:47,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/4f901353fd144a0698d7cd3405e5fe00 is 50, key is test_row_0/B:col10/1731364006251/Put/seqid=0 2024-11-11T22:26:47,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742135_1311 (size=12301) 2024-11-11T22:26:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364067553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364067563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364067563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364067563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:47,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/4f901353fd144a0698d7cd3405e5fe00 2024-11-11T22:26:47,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/4b6f1162878640249645bf760498c16b is 50, key is test_row_0/C:col10/1731364006251/Put/seqid=0 2024-11-11T22:26:47,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742136_1312 (size=12301) 2024-11-11T22:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:26:47,783 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:47,784 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T22:26:47,786 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:26:47,786 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:47,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:26:47,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T22:26:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:47,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:48,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364068063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:48,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364068073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:48,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364068073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:48,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364068083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:48,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T22:26:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:48,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:26:48,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/4b6f1162878640249645bf760498c16b 2024-11-11T22:26:48,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/be079ede7c914204aa0f99ad82e9e24e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be079ede7c914204aa0f99ad82e9e24e 2024-11-11T22:26:48,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be079ede7c914204aa0f99ad82e9e24e, entries=150, sequenceid=365, filesize=12.0 K 2024-11-11T22:26:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/4f901353fd144a0698d7cd3405e5fe00 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f901353fd144a0698d7cd3405e5fe00 2024-11-11T22:26:48,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f901353fd144a0698d7cd3405e5fe00, entries=150, sequenceid=365, filesize=12.0 K 2024-11-11T22:26:48,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/4b6f1162878640249645bf760498c16b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/4b6f1162878640249645bf760498c16b 2024-11-11T22:26:48,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/4b6f1162878640249645bf760498c16b, entries=150, sequenceid=365, filesize=12.0 K 2024-11-11T22:26:48,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 1288ms, sequenceid=365, compaction requested=true 2024-11-11T22:26:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:48,184 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:48,184 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:48,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:48,185 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:48,185 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:48,185 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/B is initiating minor compaction (all files) 2024-11-11T22:26:48,185 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/A is initiating minor compaction (all files) 2024-11-11T22:26:48,185 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/A in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:48,185 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/dbac5035a5204f66b264eadc81d91d3f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/396bf78d78ac49ae9b03cba17f662626, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/36cb951050b14525ac130d2939153355, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be079ede7c914204aa0f99ad82e9e24e] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=51.2 K 2024-11-11T22:26:48,186 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/B in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:48,186 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/464b5c444f104616ab5adc85088f85fa, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/1be5000d16544e5aa4bce4205771e2c7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b9aa03f1baef468baeac6aa708176614, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f901353fd144a0698d7cd3405e5fe00] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=48.8 K 2024-11-11T22:26:48,186 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbac5035a5204f66b264eadc81d91d3f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731364004870 2024-11-11T22:26:48,186 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 464b5c444f104616ab5adc85088f85fa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731364004870 2024-11-11T22:26:48,186 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 396bf78d78ac49ae9b03cba17f662626, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731364005272 2024-11-11T22:26:48,186 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 1be5000d16544e5aa4bce4205771e2c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731364005272 2024-11-11T22:26:48,186 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36cb951050b14525ac130d2939153355, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731364005613 2024-11-11T22:26:48,186 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting b9aa03f1baef468baeac6aa708176614, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731364005613 2024-11-11T22:26:48,186 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting be079ede7c914204aa0f99ad82e9e24e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731364006251 2024-11-11T22:26:48,187 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f901353fd144a0698d7cd3405e5fe00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731364006251 2024-11-11T22:26:48,196 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:48,197 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/ab70e450240b4aac992c33eecf52df5f is 50, key is test_row_0/B:col10/1731364006251/Put/seqid=0 2024-11-11T22:26:48,201 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:48,202 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/67c2647f62f7424a9bc363ce7a36fe84 is 50, key is test_row_0/A:col10/1731364006251/Put/seqid=0 2024-11-11T22:26:48,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742137_1313 (size=13187) 2024-11-11T22:26:48,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742138_1314 (size=13187) 2024-11-11T22:26:48,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T22:26:48,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:48,245 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:26:48,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/befe9de67bc74ed6b873f9b442677879 is 50, key is test_row_0/A:col10/1731364006925/Put/seqid=0 2024-11-11T22:26:48,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742139_1315 (size=12301) 2024-11-11T22:26:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:26:48,608 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/ab70e450240b4aac992c33eecf52df5f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/ab70e450240b4aac992c33eecf52df5f 2024-11-11T22:26:48,619 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/67c2647f62f7424a9bc363ce7a36fe84 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/67c2647f62f7424a9bc363ce7a36fe84 2024-11-11T22:26:48,620 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/B of 8b5cc6a2f2e03a352bea5a20a4c912ce into ab70e450240b4aac992c33eecf52df5f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:48,620 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:48,620 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/B, priority=12, startTime=1731364008184; duration=0sec 2024-11-11T22:26:48,621 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:48,621 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:B 2024-11-11T22:26:48,621 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:26:48,622 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:26:48,622 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/C is initiating minor compaction (all files) 2024-11-11T22:26:48,622 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/C in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:48,622 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/7e886a378074411f90e89d9f2162f9b3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/0b87ee0cd1b04acc98ebb0d9ad730f60, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/204cfe04159147938f5dbaad4f96aabf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/4b6f1162878640249645bf760498c16b] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=48.8 K 2024-11-11T22:26:48,623 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e886a378074411f90e89d9f2162f9b3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731364004870 2024-11-11T22:26:48,623 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b87ee0cd1b04acc98ebb0d9ad730f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731364005272 2024-11-11T22:26:48,623 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 204cfe04159147938f5dbaad4f96aabf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731364005613 2024-11-11T22:26:48,623 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b6f1162878640249645bf760498c16b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731364006251 2024-11-11T22:26:48,624 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/A of 8b5cc6a2f2e03a352bea5a20a4c912ce into 67c2647f62f7424a9bc363ce7a36fe84(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:48,624 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:48,624 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/A, priority=12, startTime=1731364008184; duration=0sec 2024-11-11T22:26:48,624 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:48,624 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:A 2024-11-11T22:26:48,644 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:48,645 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/c335f8a2bcc146e2afefc3ea9ebb2bf1 is 50, key is test_row_0/C:col10/1731364006251/Put/seqid=0 2024-11-11T22:26:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:48,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:48,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742140_1316 (size=13187) 2024-11-11T22:26:48,662 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/befe9de67bc74ed6b873f9b442677879 2024-11-11T22:26:48,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/06f9d448b26645ab92abed32a0f9161f is 50, key is test_row_0/B:col10/1731364006925/Put/seqid=0 2024-11-11T22:26:48,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742141_1317 (size=12301) 2024-11-11T22:26:48,673 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/06f9d448b26645ab92abed32a0f9161f 2024-11-11T22:26:48,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/1edb15e7ce7f4ba99213867e3d2fa7aa is 50, key is test_row_0/C:col10/1731364006925/Put/seqid=0 2024-11-11T22:26:48,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742142_1318 (size=12301) 2024-11-11T22:26:48,685 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/1edb15e7ce7f4ba99213867e3d2fa7aa 2024-11-11T22:26:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364068685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/befe9de67bc74ed6b873f9b442677879 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/befe9de67bc74ed6b873f9b442677879 2024-11-11T22:26:48,695 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/befe9de67bc74ed6b873f9b442677879, entries=150, sequenceid=388, filesize=12.0 K 2024-11-11T22:26:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/06f9d448b26645ab92abed32a0f9161f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/06f9d448b26645ab92abed32a0f9161f 2024-11-11T22:26:48,700 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/06f9d448b26645ab92abed32a0f9161f, entries=150, sequenceid=388, filesize=12.0 K 2024-11-11T22:26:48,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/1edb15e7ce7f4ba99213867e3d2fa7aa as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/1edb15e7ce7f4ba99213867e3d2fa7aa 2024-11-11T22:26:48,705 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/1edb15e7ce7f4ba99213867e3d2fa7aa, entries=150, sequenceid=388, filesize=12.0 K 2024-11-11T22:26:48,706 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 460ms, sequenceid=388, compaction requested=false 2024-11-11T22:26:48,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:48,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:48,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T22:26:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T22:26:48,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T22:26:48,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 921 msec 2024-11-11T22:26:48,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 924 msec 2024-11-11T22:26:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:48,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:48,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:26:48,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/5692736f34f24f0085652951dbb34625 is 50, key is test_row_0/A:col10/1731364008683/Put/seqid=0 2024-11-11T22:26:48,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742143_1319 (size=12301) 2024-11-11T22:26:48,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364068871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:26:48,924 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:48,924 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-11T22:26:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:26:48,926 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:48,926 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:48,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:48,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364068983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:26:49,058 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/c335f8a2bcc146e2afefc3ea9ebb2bf1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/c335f8a2bcc146e2afefc3ea9ebb2bf1 2024-11-11T22:26:49,063 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/C of 8b5cc6a2f2e03a352bea5a20a4c912ce into c335f8a2bcc146e2afefc3ea9ebb2bf1(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:49,063 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:49,063 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/C, priority=12, startTime=1731364008184; duration=0sec 2024-11-11T22:26:49,063 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:49,063 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:C 2024-11-11T22:26:49,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1731364069073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:49,077 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:49,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:26:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58764 deadline: 1731364069083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:49,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58772 deadline: 1731364069083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:49,087 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:49,088 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:49,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58722 deadline: 1731364069093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:49,095 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 8b5cc6a2f2e03a352bea5a20a4c912ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:49,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364069193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:49,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/5692736f34f24f0085652951dbb34625 2024-11-11T22:26:49,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/78a6106225404011b6fe16fa6ff215c2 is 50, key is test_row_0/B:col10/1731364008683/Put/seqid=0 2024-11-11T22:26:49,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742144_1320 (size=12301) 2024-11-11T22:26:49,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/78a6106225404011b6fe16fa6ff215c2 2024-11-11T22:26:49,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:26:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/238cbcfc66da4ca984141ff2e486db57 is 50, key is test_row_0/C:col10/1731364008683/Put/seqid=0 2024-11-11T22:26:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:26:49,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742145_1321 (size=12301) 2024-11-11T22:26:49,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:26:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731364069503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:26:49,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:26:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:26:49,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/238cbcfc66da4ca984141ff2e486db57 2024-11-11T22:26:49,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/5692736f34f24f0085652951dbb34625 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5692736f34f24f0085652951dbb34625 2024-11-11T22:26:49,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5692736f34f24f0085652951dbb34625, entries=150, sequenceid=404, filesize=12.0 K 2024-11-11T22:26:49,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/78a6106225404011b6fe16fa6ff215c2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/78a6106225404011b6fe16fa6ff215c2 2024-11-11T22:26:49,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/78a6106225404011b6fe16fa6ff215c2, entries=150, sequenceid=404, filesize=12.0 K 2024-11-11T22:26:49,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/238cbcfc66da4ca984141ff2e486db57 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/238cbcfc66da4ca984141ff2e486db57 2024-11-11T22:26:49,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/238cbcfc66da4ca984141ff2e486db57, entries=150, sequenceid=404, filesize=12.0 K 2024-11-11T22:26:49,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 880ms, sequenceid=404, compaction requested=true 2024-11-11T22:26:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:49,675 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:49,675 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b5cc6a2f2e03a352bea5a20a4c912ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:49,676 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:49,676 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:49,676 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/B is initiating minor compaction (all files) 2024-11-11T22:26:49,676 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/A is initiating minor compaction (all files) 2024-11-11T22:26:49,676 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/A in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,677 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-11T22:26:49,677 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/67c2647f62f7424a9bc363ce7a36fe84, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/befe9de67bc74ed6b873f9b442677879, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5692736f34f24f0085652951dbb34625] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=36.9 K 2024-11-11T22:26:49,677 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:49,677 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/B in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,677 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:49,677 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/ab70e450240b4aac992c33eecf52df5f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/06f9d448b26645ab92abed32a0f9161f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/78a6106225404011b6fe16fa6ff215c2] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=36.9 K 2024-11-11T22:26:49,677 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:49,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:49,677 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67c2647f62f7424a9bc363ce7a36fe84, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731364006251 2024-11-11T22:26:49,677 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting ab70e450240b4aac992c33eecf52df5f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731364006251 2024-11-11T22:26:49,677 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting befe9de67bc74ed6b873f9b442677879, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731364006907 2024-11-11T22:26:49,677 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-11T22:26:49,677 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 06f9d448b26645ab92abed32a0f9161f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731364006907 2024-11-11T22:26:49,677 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:49,678 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:49,678 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:49,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:49,678 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-11T22:26:49,678 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:49,678 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:49,678 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:49,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:49,678 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 78a6106225404011b6fe16fa6ff215c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731364008674 2024-11-11T22:26:49,678 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5692736f34f24f0085652951dbb34625, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731364008674 2024-11-11T22:26:49,679 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-11-11T22:26:49,679 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:49,679 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:49,680 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:49,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:49,680 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-11-11T22:26:49,680 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:49,680 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:49,680 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:49,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:49,683 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#A#compaction#279 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:49,684 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#B#compaction#280 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:49,684 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/5b6fd58a4b254b2dbffa00c5d2dd0c9f is 50, key is test_row_0/A:col10/1731364008683/Put/seqid=0 2024-11-11T22:26:49,684 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/267e31a72eef4449a0517507e8d9ac1f is 50, key is test_row_0/B:col10/1731364008683/Put/seqid=0 2024-11-11T22:26:49,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742146_1322 (size=13289) 2024-11-11T22:26:49,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742147_1323 (size=13289) 2024-11-11T22:26:49,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:26:49,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:49,697 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T22:26:49,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/1ae53a4302b5422a815424f3907e6942 is 50, key is test_row_0/A:col10/1731364008861/Put/seqid=0 2024-11-11T22:26:49,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742148_1324 (size=12301) 2024-11-11T22:26:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:50,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. as already flushing 2024-11-11T22:26:50,015 INFO [Thread-1098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1098. 2024-11-11T22:26:50,015 DEBUG [Thread-1098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:50,015 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:50,015 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:50,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:26:50,091 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/267e31a72eef4449a0517507e8d9ac1f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/267e31a72eef4449a0517507e8d9ac1f 2024-11-11T22:26:50,091 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/5b6fd58a4b254b2dbffa00c5d2dd0c9f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5b6fd58a4b254b2dbffa00c5d2dd0c9f 2024-11-11T22:26:50,095 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/A of 8b5cc6a2f2e03a352bea5a20a4c912ce into 5b6fd58a4b254b2dbffa00c5d2dd0c9f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:50,095 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/B of 8b5cc6a2f2e03a352bea5a20a4c912ce into 267e31a72eef4449a0517507e8d9ac1f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:50,095 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:50,095 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:50,095 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/A, priority=13, startTime=1731364009675; duration=0sec 2024-11-11T22:26:50,095 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/B, priority=13, startTime=1731364009675; duration=0sec 2024-11-11T22:26:50,095 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:50,095 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:50,095 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:A 2024-11-11T22:26:50,095 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:B 2024-11-11T22:26:50,095 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:50,096 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:50,096 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 8b5cc6a2f2e03a352bea5a20a4c912ce/C is initiating minor compaction (all files) 2024-11-11T22:26:50,096 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b5cc6a2f2e03a352bea5a20a4c912ce/C in TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:50,096 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/c335f8a2bcc146e2afefc3ea9ebb2bf1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/1edb15e7ce7f4ba99213867e3d2fa7aa, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/238cbcfc66da4ca984141ff2e486db57] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp, totalSize=36.9 K 2024-11-11T22:26:50,096 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting c335f8a2bcc146e2afefc3ea9ebb2bf1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731364006251 2024-11-11T22:26:50,097 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1edb15e7ce7f4ba99213867e3d2fa7aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731364006907 2024-11-11T22:26:50,097 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 238cbcfc66da4ca984141ff2e486db57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731364008674 2024-11-11T22:26:50,102 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b5cc6a2f2e03a352bea5a20a4c912ce#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:50,102 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/91e8c30051de422182c5120e7249976d is 50, key is test_row_0/C:col10/1731364008683/Put/seqid=0 2024-11-11T22:26:50,104 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/1ae53a4302b5422a815424f3907e6942 2024-11-11T22:26:50,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742149_1325 (size=13289) 2024-11-11T22:26:50,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/f8ce50f9d8a54986b6c85791cb062b98 is 50, key is test_row_0/B:col10/1731364008861/Put/seqid=0 2024-11-11T22:26:50,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742150_1326 (size=12301) 2024-11-11T22:26:50,510 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/91e8c30051de422182c5120e7249976d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/91e8c30051de422182c5120e7249976d 2024-11-11T22:26:50,512 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/f8ce50f9d8a54986b6c85791cb062b98 2024-11-11T22:26:50,515 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b5cc6a2f2e03a352bea5a20a4c912ce/C of 8b5cc6a2f2e03a352bea5a20a4c912ce into 91e8c30051de422182c5120e7249976d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:26:50,515 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:50,515 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., storeName=8b5cc6a2f2e03a352bea5a20a4c912ce/C, priority=13, startTime=1731364009675; duration=0sec 2024-11-11T22:26:50,515 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:50,515 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b5cc6a2f2e03a352bea5a20a4c912ce:C 2024-11-11T22:26:50,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/785fa50fc2414b728391f69726ce78c7 is 50, key is test_row_0/C:col10/1731364008861/Put/seqid=0 2024-11-11T22:26:50,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742151_1327 (size=12301) 2024-11-11T22:26:50,921 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/785fa50fc2414b728391f69726ce78c7 2024-11-11T22:26:50,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/1ae53a4302b5422a815424f3907e6942 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/1ae53a4302b5422a815424f3907e6942 2024-11-11T22:26:50,930 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/1ae53a4302b5422a815424f3907e6942, entries=150, sequenceid=427, filesize=12.0 K 2024-11-11T22:26:50,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/f8ce50f9d8a54986b6c85791cb062b98 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f8ce50f9d8a54986b6c85791cb062b98 2024-11-11T22:26:50,935 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f8ce50f9d8a54986b6c85791cb062b98, entries=150, sequenceid=427, filesize=12.0 K 2024-11-11T22:26:50,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/785fa50fc2414b728391f69726ce78c7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/785fa50fc2414b728391f69726ce78c7 2024-11-11T22:26:50,940 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/785fa50fc2414b728391f69726ce78c7, entries=150, sequenceid=427, filesize=12.0 K 2024-11-11T22:26:50,941 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=6.71 KB/6870 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 1244ms, sequenceid=427, compaction requested=false 2024-11-11T22:26:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: 2024-11-11T22:26:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-11T22:26:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-11T22:26:50,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T22:26:50,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0160 sec 2024-11-11T22:26:50,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.0200 sec 2024-11-11T22:26:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:26:51,073 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:51,095 INFO [Thread-1100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1100. 2024-11-11T22:26:51,095 DEBUG [Thread-1100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:51,095 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:51,095 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:51,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:51,105 INFO [Thread-1096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1096. 2024-11-11T22:26:51,105 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-11-11T22:26:51,105 DEBUG [Thread-1096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:51,105 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:51,105 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:51,105 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:51,105 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:51,105 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:51,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:51,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:51,115 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-11-11T22:26:51,115 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:26:51,115 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:51,115 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:51,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:51,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:26:51,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-11T22:26:51,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-11T22:26:51,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-11T22:26:51,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-11T22:26:51,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 167 2024-11-11T22:26:51,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:26:51,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3938 2024-11-11T22:26:51,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3886 2024-11-11T22:26:51,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3890 2024-11-11T22:26:51,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3914 2024-11-11T22:26:51,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3878 2024-11-11T22:26:51,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:26:51,116 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:26:51,116 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:26:51,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:51,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:51,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:26:51,117 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:26:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:26:51,120 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364011120"}]},"ts":"1731364011120"} 2024-11-11T22:26:51,121 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:26:51,121 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:26:51,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:26:51,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b5cc6a2f2e03a352bea5a20a4c912ce, UNASSIGN}] 2024-11-11T22:26:51,124 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b5cc6a2f2e03a352bea5a20a4c912ce, UNASSIGN 2024-11-11T22:26:51,125 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=8b5cc6a2f2e03a352bea5a20a4c912ce, regionState=CLOSING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:51,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b5cc6a2f2e03a352bea5a20a4c912ce, UNASSIGN because future has completed 2024-11-11T22:26:51,126 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:26:51,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:26:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:26:51,278 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:51,279 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:26:51,279 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 8b5cc6a2f2e03a352bea5a20a4c912ce, disabling compactions & flushes 2024-11-11T22:26:51,279 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:51,279 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:51,279 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. after waiting 0 ms 2024-11-11T22:26:51,279 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:51,279 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T22:26:51,283 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/849f0de5ec1f4ef1bd4b7293b8a22b00 is 50, key is test_row_0/A:col10/1731364011113/Put/seqid=0 2024-11-11T22:26:51,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742152_1328 (size=12301) 2024-11-11T22:26:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:26:51,688 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/849f0de5ec1f4ef1bd4b7293b8a22b00 2024-11-11T22:26:51,696 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/cb120fbc93454713adc8b0fcaeb718b2 is 50, key is test_row_0/B:col10/1731364011113/Put/seqid=0 2024-11-11T22:26:51,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742153_1329 (size=12301) 2024-11-11T22:26:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:26:52,101 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/cb120fbc93454713adc8b0fcaeb718b2 2024-11-11T22:26:52,110 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d0a07c9e768b4bbcbf1d0f57fe66f731 is 50, key is test_row_0/C:col10/1731364011113/Put/seqid=0 2024-11-11T22:26:52,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742154_1330 (size=12301) 2024-11-11T22:26:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:26:52,514 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d0a07c9e768b4bbcbf1d0f57fe66f731 2024-11-11T22:26:52,519 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/A/849f0de5ec1f4ef1bd4b7293b8a22b00 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/849f0de5ec1f4ef1bd4b7293b8a22b00 2024-11-11T22:26:52,523 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/849f0de5ec1f4ef1bd4b7293b8a22b00, entries=150, sequenceid=438, filesize=12.0 K 2024-11-11T22:26:52,524 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/B/cb120fbc93454713adc8b0fcaeb718b2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/cb120fbc93454713adc8b0fcaeb718b2 2024-11-11T22:26:52,527 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/cb120fbc93454713adc8b0fcaeb718b2, entries=150, sequenceid=438, filesize=12.0 K 2024-11-11T22:26:52,528 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/.tmp/C/d0a07c9e768b4bbcbf1d0f57fe66f731 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d0a07c9e768b4bbcbf1d0f57fe66f731 2024-11-11T22:26:52,531 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d0a07c9e768b4bbcbf1d0f57fe66f731, entries=150, sequenceid=438, filesize=12.0 K 2024-11-11T22:26:52,532 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 1253ms, sequenceid=438, compaction requested=true 2024-11-11T22:26:52,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/fc321eda01a840eb81e68340f06b712f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/8155ecdaef5541d4a8c4aeea5656c2cc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/255ec97985b54439bcef81019cc47e84, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/15fd28e050684e3ebdb923c5709a0d06, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b9e354f6f8364d079c018cdb1f4950d9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9a52af2af51d45d0bccf62133c1768ab, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/125913d6ded54f45bc2c3e7ee362f4e9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/748c0d4e23e04a8eb3513e89f9535d6b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b277b813afea43979afed6cffbb1e6f7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/87a439e92485422fa807e4756a0269b1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9357f32e716d49f3aa3ab7d906cf4d10, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/2dc4a68677a74f10b7ce6da93dcff92a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ace6cceaeb844ab4b531fa4ca6f30bb3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be708e90410349f1b0e9117d0f12ee7e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/10e6de13d6494ce88cf4618100797c59, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/513e7adde3e248e58a12c437bd3a1666, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/d41deac6d468479f88dcbc8f5bd3817e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ed4741f71bcf43e190aa9bdaff71ccb9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5a57ed8313ef45ffae4ee79ad088d547, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/28a82fedd8ca4bada70cfe3f1d1dfd0f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/00b78e9e169f445b881f5b4caa09421f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b511188062294df8bef69b579843ad44, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/dbac5035a5204f66b264eadc81d91d3f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/396bf78d78ac49ae9b03cba17f662626, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/36cb951050b14525ac130d2939153355, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/67c2647f62f7424a9bc363ce7a36fe84, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be079ede7c914204aa0f99ad82e9e24e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/befe9de67bc74ed6b873f9b442677879, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5692736f34f24f0085652951dbb34625] to archive 2024-11-11T22:26:52,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:26:52,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/fc321eda01a840eb81e68340f06b712f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/fc321eda01a840eb81e68340f06b712f 2024-11-11T22:26:52,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/8155ecdaef5541d4a8c4aeea5656c2cc to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/8155ecdaef5541d4a8c4aeea5656c2cc 2024-11-11T22:26:52,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/255ec97985b54439bcef81019cc47e84 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/255ec97985b54439bcef81019cc47e84 2024-11-11T22:26:52,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/15fd28e050684e3ebdb923c5709a0d06 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/15fd28e050684e3ebdb923c5709a0d06 2024-11-11T22:26:52,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b9e354f6f8364d079c018cdb1f4950d9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b9e354f6f8364d079c018cdb1f4950d9 2024-11-11T22:26:52,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9a52af2af51d45d0bccf62133c1768ab to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9a52af2af51d45d0bccf62133c1768ab 2024-11-11T22:26:52,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/125913d6ded54f45bc2c3e7ee362f4e9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/125913d6ded54f45bc2c3e7ee362f4e9 2024-11-11T22:26:52,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/748c0d4e23e04a8eb3513e89f9535d6b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/748c0d4e23e04a8eb3513e89f9535d6b 2024-11-11T22:26:52,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b277b813afea43979afed6cffbb1e6f7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b277b813afea43979afed6cffbb1e6f7 2024-11-11T22:26:52,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/87a439e92485422fa807e4756a0269b1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/87a439e92485422fa807e4756a0269b1 2024-11-11T22:26:52,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9357f32e716d49f3aa3ab7d906cf4d10 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/9357f32e716d49f3aa3ab7d906cf4d10 2024-11-11T22:26:52,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/2dc4a68677a74f10b7ce6da93dcff92a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/2dc4a68677a74f10b7ce6da93dcff92a 2024-11-11T22:26:52,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ace6cceaeb844ab4b531fa4ca6f30bb3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ace6cceaeb844ab4b531fa4ca6f30bb3 2024-11-11T22:26:52,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be708e90410349f1b0e9117d0f12ee7e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be708e90410349f1b0e9117d0f12ee7e 2024-11-11T22:26:52,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/10e6de13d6494ce88cf4618100797c59 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/10e6de13d6494ce88cf4618100797c59 2024-11-11T22:26:52,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/513e7adde3e248e58a12c437bd3a1666 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/513e7adde3e248e58a12c437bd3a1666 2024-11-11T22:26:52,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/d41deac6d468479f88dcbc8f5bd3817e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/d41deac6d468479f88dcbc8f5bd3817e 2024-11-11T22:26:52,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ed4741f71bcf43e190aa9bdaff71ccb9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/ed4741f71bcf43e190aa9bdaff71ccb9 2024-11-11T22:26:52,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5a57ed8313ef45ffae4ee79ad088d547 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5a57ed8313ef45ffae4ee79ad088d547 2024-11-11T22:26:52,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/28a82fedd8ca4bada70cfe3f1d1dfd0f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/28a82fedd8ca4bada70cfe3f1d1dfd0f 2024-11-11T22:26:52,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/00b78e9e169f445b881f5b4caa09421f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/00b78e9e169f445b881f5b4caa09421f 2024-11-11T22:26:52,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b511188062294df8bef69b579843ad44 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/b511188062294df8bef69b579843ad44 2024-11-11T22:26:52,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/dbac5035a5204f66b264eadc81d91d3f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/dbac5035a5204f66b264eadc81d91d3f 2024-11-11T22:26:52,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/396bf78d78ac49ae9b03cba17f662626 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/396bf78d78ac49ae9b03cba17f662626 2024-11-11T22:26:52,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/36cb951050b14525ac130d2939153355 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/36cb951050b14525ac130d2939153355 2024-11-11T22:26:52,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/67c2647f62f7424a9bc363ce7a36fe84 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/67c2647f62f7424a9bc363ce7a36fe84 2024-11-11T22:26:52,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be079ede7c914204aa0f99ad82e9e24e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/be079ede7c914204aa0f99ad82e9e24e 2024-11-11T22:26:52,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/befe9de67bc74ed6b873f9b442677879 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/befe9de67bc74ed6b873f9b442677879 2024-11-11T22:26:52,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5692736f34f24f0085652951dbb34625 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5692736f34f24f0085652951dbb34625 2024-11-11T22:26:52,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/08f0128870bb4a3bbaa9cd65057aaaed, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/8b1cb0dafdea4fd6bdc6c691e7cc79ed, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b8b76d8baade4adfbb7208fec7fdccf8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f233c7ec89d14451aecdd9702ca26b90, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/43599986061a401fa3f33dec77980910, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f1dbd7052f02493eb542109a7347dc83, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/92b251f6b06f4112a7b2fd2258061150, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27b6df22d3ad43f183df2d4cf5047040, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/666bd9e5ba6d4c5a90dfad0a61eac3cd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27d8d9a9348245c596b9047c9c3e55e8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/6e97f66325314231bed74b4d7df5ff24, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/5ec090771fe946efb5d08cc888e640de, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0252767e6dae463d93742e0fe6a63fbf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0db792651abb40d3a62ff071269d1937, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/36b98daea5f447629bddaa108f575739, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0a12d02129a94da1b78d539d99147431, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/575c5fce6a1e46aa9498fc3ff3843c19, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f2e7401c40443a08579f1ff478c1417, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/68b733684b24493ca1ca0c6a76de44ee, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/475fd4b28f6449a5bb0003d6a22408bf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/9e1a4469c2004f658e55f4aebf4cce3c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/464b5c444f104616ab5adc85088f85fa, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/a6c3e495515a4a659ac02e20a129b659, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/1be5000d16544e5aa4bce4205771e2c7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b9aa03f1baef468baeac6aa708176614, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/ab70e450240b4aac992c33eecf52df5f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f901353fd144a0698d7cd3405e5fe00, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/06f9d448b26645ab92abed32a0f9161f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/78a6106225404011b6fe16fa6ff215c2] to archive 2024-11-11T22:26:52,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:26:52,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/08f0128870bb4a3bbaa9cd65057aaaed to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/08f0128870bb4a3bbaa9cd65057aaaed 2024-11-11T22:26:52,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/8b1cb0dafdea4fd6bdc6c691e7cc79ed to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/8b1cb0dafdea4fd6bdc6c691e7cc79ed 2024-11-11T22:26:52,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b8b76d8baade4adfbb7208fec7fdccf8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b8b76d8baade4adfbb7208fec7fdccf8 2024-11-11T22:26:52,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f233c7ec89d14451aecdd9702ca26b90 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f233c7ec89d14451aecdd9702ca26b90 2024-11-11T22:26:52,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/43599986061a401fa3f33dec77980910 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/43599986061a401fa3f33dec77980910 2024-11-11T22:26:52,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f1dbd7052f02493eb542109a7347dc83 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f1dbd7052f02493eb542109a7347dc83 2024-11-11T22:26:52,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/92b251f6b06f4112a7b2fd2258061150 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/92b251f6b06f4112a7b2fd2258061150 2024-11-11T22:26:52,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27b6df22d3ad43f183df2d4cf5047040 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27b6df22d3ad43f183df2d4cf5047040 2024-11-11T22:26:52,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/666bd9e5ba6d4c5a90dfad0a61eac3cd to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/666bd9e5ba6d4c5a90dfad0a61eac3cd 2024-11-11T22:26:52,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27d8d9a9348245c596b9047c9c3e55e8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/27d8d9a9348245c596b9047c9c3e55e8 2024-11-11T22:26:52,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/6e97f66325314231bed74b4d7df5ff24 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/6e97f66325314231bed74b4d7df5ff24 2024-11-11T22:26:52,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/5ec090771fe946efb5d08cc888e640de to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/5ec090771fe946efb5d08cc888e640de 2024-11-11T22:26:52,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0252767e6dae463d93742e0fe6a63fbf to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0252767e6dae463d93742e0fe6a63fbf 2024-11-11T22:26:52,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0db792651abb40d3a62ff071269d1937 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0db792651abb40d3a62ff071269d1937 2024-11-11T22:26:52,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/36b98daea5f447629bddaa108f575739 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/36b98daea5f447629bddaa108f575739 2024-11-11T22:26:52,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0a12d02129a94da1b78d539d99147431 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/0a12d02129a94da1b78d539d99147431 2024-11-11T22:26:52,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/575c5fce6a1e46aa9498fc3ff3843c19 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/575c5fce6a1e46aa9498fc3ff3843c19 2024-11-11T22:26:52,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f2e7401c40443a08579f1ff478c1417 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f2e7401c40443a08579f1ff478c1417 2024-11-11T22:26:52,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/68b733684b24493ca1ca0c6a76de44ee to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/68b733684b24493ca1ca0c6a76de44ee 2024-11-11T22:26:52,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/475fd4b28f6449a5bb0003d6a22408bf to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/475fd4b28f6449a5bb0003d6a22408bf 2024-11-11T22:26:52,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/9e1a4469c2004f658e55f4aebf4cce3c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/9e1a4469c2004f658e55f4aebf4cce3c 2024-11-11T22:26:52,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/464b5c444f104616ab5adc85088f85fa to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/464b5c444f104616ab5adc85088f85fa 2024-11-11T22:26:52,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/a6c3e495515a4a659ac02e20a129b659 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/a6c3e495515a4a659ac02e20a129b659 2024-11-11T22:26:52,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/1be5000d16544e5aa4bce4205771e2c7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/1be5000d16544e5aa4bce4205771e2c7 2024-11-11T22:26:52,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b9aa03f1baef468baeac6aa708176614 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/b9aa03f1baef468baeac6aa708176614 2024-11-11T22:26:52,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/ab70e450240b4aac992c33eecf52df5f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/ab70e450240b4aac992c33eecf52df5f 2024-11-11T22:26:52,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f901353fd144a0698d7cd3405e5fe00 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/4f901353fd144a0698d7cd3405e5fe00 2024-11-11T22:26:52,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/06f9d448b26645ab92abed32a0f9161f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/06f9d448b26645ab92abed32a0f9161f 2024-11-11T22:26:52,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/78a6106225404011b6fe16fa6ff215c2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/78a6106225404011b6fe16fa6ff215c2 2024-11-11T22:26:52,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/f8e9480239c846e0ade4aff91c2b12e8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/259cc43ff0594061b0735342aae47e2f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/7e1f5115410c4dd18797c23b6d2663ab, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bd5b462f3e214d2caf2bd10b40473ded, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/5ce8741feb2e4fdcacb968e5e50a0e83, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bcde66a8c4014ba7ad59c94a5b1ade7b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/fe933be0370d41df97eecfe1e46d7e02, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8be9c73654964346a73e59268246776b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/31767ac4994d4597b89f9f43203dea7b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/035fd32916a84bd49603f8554f4c88f3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/78c5cb5033ce4583946efeb447801efd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/75b9f5ab76db42859a1f9bf8a7666989, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/99154d6732c04fc4a8aeb9df8d80ca1b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d3d6e3f7ebfe454bae7d818cf7fb5bc7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/45e839a83c6442cfb71bbcaec444cdfc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/213cb5c3ccdb41e1a1c460caacd3e5ae, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/918b53b4f78a43e3b6dc3a2c8d9b529f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/cf49cce69d9d4a3cbaf34f42a7705acb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8f73bba6d2d349adb2a28478f51f5879, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d2df0486a082410fa2b84cc9940afc06, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/563118c861cb4bcaaff69e858a6d8412, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/7e886a378074411f90e89d9f2162f9b3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d04fa4fba7934016ba40448648f81c45, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/0b87ee0cd1b04acc98ebb0d9ad730f60, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/204cfe04159147938f5dbaad4f96aabf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/c335f8a2bcc146e2afefc3ea9ebb2bf1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/4b6f1162878640249645bf760498c16b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/1edb15e7ce7f4ba99213867e3d2fa7aa, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/238cbcfc66da4ca984141ff2e486db57] to archive 2024-11-11T22:26:52,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:26:52,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/f8e9480239c846e0ade4aff91c2b12e8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/f8e9480239c846e0ade4aff91c2b12e8 2024-11-11T22:26:52,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/259cc43ff0594061b0735342aae47e2f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/259cc43ff0594061b0735342aae47e2f 2024-11-11T22:26:52,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/7e1f5115410c4dd18797c23b6d2663ab to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/7e1f5115410c4dd18797c23b6d2663ab 2024-11-11T22:26:52,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bd5b462f3e214d2caf2bd10b40473ded to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bd5b462f3e214d2caf2bd10b40473ded 2024-11-11T22:26:52,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/5ce8741feb2e4fdcacb968e5e50a0e83 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/5ce8741feb2e4fdcacb968e5e50a0e83 2024-11-11T22:26:52,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bcde66a8c4014ba7ad59c94a5b1ade7b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/bcde66a8c4014ba7ad59c94a5b1ade7b 2024-11-11T22:26:52,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/fe933be0370d41df97eecfe1e46d7e02 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/fe933be0370d41df97eecfe1e46d7e02 2024-11-11T22:26:52,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8be9c73654964346a73e59268246776b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8be9c73654964346a73e59268246776b 2024-11-11T22:26:52,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/31767ac4994d4597b89f9f43203dea7b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/31767ac4994d4597b89f9f43203dea7b 2024-11-11T22:26:52,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/035fd32916a84bd49603f8554f4c88f3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/035fd32916a84bd49603f8554f4c88f3 2024-11-11T22:26:52,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/78c5cb5033ce4583946efeb447801efd to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/78c5cb5033ce4583946efeb447801efd 2024-11-11T22:26:52,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/75b9f5ab76db42859a1f9bf8a7666989 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/75b9f5ab76db42859a1f9bf8a7666989 2024-11-11T22:26:52,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/99154d6732c04fc4a8aeb9df8d80ca1b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/99154d6732c04fc4a8aeb9df8d80ca1b 2024-11-11T22:26:52,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d3d6e3f7ebfe454bae7d818cf7fb5bc7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d3d6e3f7ebfe454bae7d818cf7fb5bc7 2024-11-11T22:26:52,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/45e839a83c6442cfb71bbcaec444cdfc to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/45e839a83c6442cfb71bbcaec444cdfc 2024-11-11T22:26:52,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/213cb5c3ccdb41e1a1c460caacd3e5ae to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/213cb5c3ccdb41e1a1c460caacd3e5ae 2024-11-11T22:26:52,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/918b53b4f78a43e3b6dc3a2c8d9b529f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/918b53b4f78a43e3b6dc3a2c8d9b529f 2024-11-11T22:26:52,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/cf49cce69d9d4a3cbaf34f42a7705acb to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/cf49cce69d9d4a3cbaf34f42a7705acb 2024-11-11T22:26:52,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8f73bba6d2d349adb2a28478f51f5879 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/8f73bba6d2d349adb2a28478f51f5879 2024-11-11T22:26:52,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d2df0486a082410fa2b84cc9940afc06 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d2df0486a082410fa2b84cc9940afc06 2024-11-11T22:26:52,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/563118c861cb4bcaaff69e858a6d8412 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/563118c861cb4bcaaff69e858a6d8412 2024-11-11T22:26:52,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/7e886a378074411f90e89d9f2162f9b3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/7e886a378074411f90e89d9f2162f9b3 2024-11-11T22:26:52,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d04fa4fba7934016ba40448648f81c45 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d04fa4fba7934016ba40448648f81c45 2024-11-11T22:26:52,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/0b87ee0cd1b04acc98ebb0d9ad730f60 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/0b87ee0cd1b04acc98ebb0d9ad730f60 2024-11-11T22:26:52,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/204cfe04159147938f5dbaad4f96aabf to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/204cfe04159147938f5dbaad4f96aabf 2024-11-11T22:26:52,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/c335f8a2bcc146e2afefc3ea9ebb2bf1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/c335f8a2bcc146e2afefc3ea9ebb2bf1 2024-11-11T22:26:52,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/4b6f1162878640249645bf760498c16b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/4b6f1162878640249645bf760498c16b 2024-11-11T22:26:52,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/1edb15e7ce7f4ba99213867e3d2fa7aa to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/1edb15e7ce7f4ba99213867e3d2fa7aa 2024-11-11T22:26:52,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/238cbcfc66da4ca984141ff2e486db57 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/238cbcfc66da4ca984141ff2e486db57 2024-11-11T22:26:52,624 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/recovered.edits/441.seqid, newMaxSeqId=441, maxSeqId=1 2024-11-11T22:26:52,624 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. 2024-11-11T22:26:52,624 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 8b5cc6a2f2e03a352bea5a20a4c912ce: Waiting for close lock at 1731364011279Running coprocessor pre-close hooks at 1731364011279Disabling compacts and flushes for region at 1731364011279Disabling writes for close at 1731364011279Obtaining lock to block concurrent updates at 1731364011279Preparing flush snapshotting stores in 8b5cc6a2f2e03a352bea5a20a4c912ce at 1731364011279Finished memstore snapshotting TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731364011279Flushing stores of TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce. at 1731364011280 (+1 ms)Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce/A: creating writer at 1731364011280Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce/A: appending metadata at 1731364011282 (+2 ms)Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce/A: closing flushed file at 1731364011282Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce/B: creating writer at 1731364011693 (+411 ms)Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce/B: appending metadata at 1731364011695 (+2 ms)Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce/B: closing flushed file at 1731364011696 (+1 ms)Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce/C: creating writer at 1731364012107 (+411 ms)Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce/C: appending metadata at 1731364012109 (+2 ms)Flushing 8b5cc6a2f2e03a352bea5a20a4c912ce/C: closing flushed file at 1731364012109Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4446a37f: reopening flushed file at 1731364012518 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@189cb95b: reopening flushed file at 1731364012523 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52da5892: reopening flushed file at 1731364012527 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8b5cc6a2f2e03a352bea5a20a4c912ce in 1253ms, sequenceid=438, compaction requested=true at 1731364012532 (+5 ms)Writing region close event to WAL at 1731364012621 (+89 ms)Running coprocessor post-close hooks at 1731364012624 (+3 ms)Closed at 1731364012624 2024-11-11T22:26:52,626 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:52,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=8b5cc6a2f2e03a352bea5a20a4c912ce, regionState=CLOSED 2024-11-11T22:26:52,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:26:52,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T22:26:52,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 8b5cc6a2f2e03a352bea5a20a4c912ce, server=8d82b89dd27d,39613,1731363918313 in 1.5030 sec 2024-11-11T22:26:52,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-11T22:26:52,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b5cc6a2f2e03a352bea5a20a4c912ce, UNASSIGN in 1.5070 sec 2024-11-11T22:26:52,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T22:26:52,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5100 sec 2024-11-11T22:26:52,635 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364012635"}]},"ts":"1731364012635"} 2024-11-11T22:26:52,636 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:26:52,636 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:26:52,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5200 sec 2024-11-11T22:26:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:26:53,265 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:53,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:26:53,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:53,272 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T22:26:53,273 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:53,276 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:53,279 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/recovered.edits] 2024-11-11T22:26:53,283 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/1ae53a4302b5422a815424f3907e6942 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/1ae53a4302b5422a815424f3907e6942 2024-11-11T22:26:53,285 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5b6fd58a4b254b2dbffa00c5d2dd0c9f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/5b6fd58a4b254b2dbffa00c5d2dd0c9f 2024-11-11T22:26:53,286 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/849f0de5ec1f4ef1bd4b7293b8a22b00 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/A/849f0de5ec1f4ef1bd4b7293b8a22b00 2024-11-11T22:26:53,289 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/267e31a72eef4449a0517507e8d9ac1f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/267e31a72eef4449a0517507e8d9ac1f 2024-11-11T22:26:53,290 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/cb120fbc93454713adc8b0fcaeb718b2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/cb120fbc93454713adc8b0fcaeb718b2 2024-11-11T22:26:53,291 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f8ce50f9d8a54986b6c85791cb062b98 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/B/f8ce50f9d8a54986b6c85791cb062b98 2024-11-11T22:26:53,293 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/785fa50fc2414b728391f69726ce78c7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/785fa50fc2414b728391f69726ce78c7 2024-11-11T22:26:53,293 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/91e8c30051de422182c5120e7249976d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/91e8c30051de422182c5120e7249976d 2024-11-11T22:26:53,294 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d0a07c9e768b4bbcbf1d0f57fe66f731 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/C/d0a07c9e768b4bbcbf1d0f57fe66f731 2024-11-11T22:26:53,296 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/recovered.edits/441.seqid to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce/recovered.edits/441.seqid 2024-11-11T22:26:53,296 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/8b5cc6a2f2e03a352bea5a20a4c912ce 2024-11-11T22:26:53,296 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:26:53,298 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:53,300 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:26:53,302 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:26:53,303 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:53,303 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:26:53,303 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731364013303"}]},"ts":"9223372036854775807"} 2024-11-11T22:26:53,305 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:26:53,305 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8b5cc6a2f2e03a352bea5a20a4c912ce, NAME => 'TestAcidGuarantees,,1731363987462.8b5cc6a2f2e03a352bea5a20a4c912ce.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:26:53,305 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:26:53,306 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731364013305"}]},"ts":"9223372036854775807"} 2024-11-11T22:26:53,307 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:26:53,308 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:53,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-11T22:26:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T22:26:53,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:26:53,384 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:53,395 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-97114466_22 at /127.0.0.1:49252 [Waiting for operation #338] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_437819696_22 at /127.0.0.1:44380 [Waiting for operation #268] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=512 (was 562), ProcessCount=11 (was 11), AvailableMemoryMB=5583 (was 5709) 2024-11-11T22:26:53,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=512, ProcessCount=11, AvailableMemoryMB=5583 2024-11-11T22:26:53,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:26:53,405 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:26:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:53,407 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:26:53,407 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:53,408 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-11T22:26:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T22:26:53,409 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:26:53,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742155_1331 (size=959) 2024-11-11T22:26:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T22:26:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T22:26:53,816 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e 2024-11-11T22:26:53,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742156_1332 (size=53) 2024-11-11T22:26:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T22:26:54,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:26:54,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1077b11658f2ec3dcbfd684d096e1030, disabling compactions & flushes 2024-11-11T22:26:54,222 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:54,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:54,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. after waiting 0 ms 2024-11-11T22:26:54,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:54,222 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:54,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1077b11658f2ec3dcbfd684d096e1030: Waiting for close lock at 1731364014222Disabling compacts and flushes for region at 1731364014222Disabling writes for close at 1731364014222Writing region close event to WAL at 1731364014222Closed at 1731364014222 2024-11-11T22:26:54,224 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:26:54,224 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731364014224"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731364014224"}]},"ts":"1731364014224"} 2024-11-11T22:26:54,226 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:26:54,226 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:26:54,227 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364014227"}]},"ts":"1731364014227"} 2024-11-11T22:26:54,229 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:26:54,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, ASSIGN}] 2024-11-11T22:26:54,231 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, ASSIGN 2024-11-11T22:26:54,232 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, ASSIGN; state=OFFLINE, location=8d82b89dd27d,39613,1731363918313; forceNewPlan=false, retain=false 2024-11-11T22:26:54,383 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=1077b11658f2ec3dcbfd684d096e1030, regionState=OPENING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:54,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, ASSIGN because future has completed 2024-11-11T22:26:54,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:26:54,540 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:54,540 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:26:54,540 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,540 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:26:54,540 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,540 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T22:26:54,545 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,546 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1077b11658f2ec3dcbfd684d096e1030 columnFamilyName A 2024-11-11T22:26:54,546 DEBUG [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:54,546 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(327): Store=1077b11658f2ec3dcbfd684d096e1030/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:26:54,546 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,548 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1077b11658f2ec3dcbfd684d096e1030 columnFamilyName B 2024-11-11T22:26:54,548 DEBUG [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:54,548 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(327): Store=1077b11658f2ec3dcbfd684d096e1030/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:26:54,548 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,554 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1077b11658f2ec3dcbfd684d096e1030 columnFamilyName C 2024-11-11T22:26:54,554 DEBUG [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:54,554 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(327): Store=1077b11658f2ec3dcbfd684d096e1030/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:26:54,555 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,555 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,556 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,557 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,557 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,557 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:26:54,558 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,560 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:26:54,560 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 1077b11658f2ec3dcbfd684d096e1030; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64257359, jitterRate=-0.04249073565006256}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:26:54,560 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:54,561 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 1077b11658f2ec3dcbfd684d096e1030: Running coprocessor pre-open hook at 1731364014541Writing region info on filesystem at 1731364014541Initializing all the Stores at 1731364014544 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364014544Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364014544Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364014544Cleaning up temporary data from old regions at 1731364014557 (+13 ms)Running coprocessor post-open hooks at 1731364014560 (+3 ms)Region opened successfully at 1731364014561 (+1 ms) 2024-11-11T22:26:54,562 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., pid=87, masterSystemTime=1731364014537 2024-11-11T22:26:54,563 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:54,564 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:54,565 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=1077b11658f2ec3dcbfd684d096e1030, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:54,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:26:54,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-11T22:26:54,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 in 182 msec 2024-11-11T22:26:54,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-11T22:26:54,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, ASSIGN in 339 msec 2024-11-11T22:26:54,572 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:26:54,572 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364014572"}]},"ts":"1731364014572"} 2024-11-11T22:26:54,573 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:26:54,574 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:26:54,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-11T22:26:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T22:26:55,553 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:55,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:55,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:55,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:55,556 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c9dad0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:55,558 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6641d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:55,559 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:55,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:55,561 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:26:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,40133,1731363917560 2024-11-11T22:26:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58be5f2a 2024-11-11T22:26:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:26:55,563 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:26:55,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:26:55,565 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:26:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T22:26:55,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742157_1333 (size=995) 2024-11-11T22:26:55,981 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T22:26:55,981 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T22:26:55,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:26:55,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, REOPEN/MOVE}] 2024-11-11T22:26:55,987 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, REOPEN/MOVE 2024-11-11T22:26:55,987 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=1077b11658f2ec3dcbfd684d096e1030, regionState=CLOSING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:55,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, REOPEN/MOVE because future has completed 2024-11-11T22:26:55,989 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:26:55,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:26:56,141 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,141 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:26:56,141 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 1077b11658f2ec3dcbfd684d096e1030, disabling compactions & flushes 2024-11-11T22:26:56,142 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,142 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,142 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. after waiting 0 ms 2024-11-11T22:26:56,142 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,145 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T22:26:56,146 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,146 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 1077b11658f2ec3dcbfd684d096e1030: Waiting for close lock at 1731364016141Running coprocessor pre-close hooks at 1731364016141Disabling compacts and flushes for region at 1731364016141Disabling writes for close at 1731364016142 (+1 ms)Writing region close event to WAL at 1731364016142Running coprocessor post-close hooks at 1731364016146 (+4 ms)Closed at 1731364016146 2024-11-11T22:26:56,146 WARN [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 1077b11658f2ec3dcbfd684d096e1030 to self. 2024-11-11T22:26:56,148 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,149 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=1077b11658f2ec3dcbfd684d096e1030, regionState=CLOSED 2024-11-11T22:26:56,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:26:56,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-11T22:26:56,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 in 162 msec 2024-11-11T22:26:56,153 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, REOPEN/MOVE; state=CLOSED, location=8d82b89dd27d,39613,1731363918313; forceNewPlan=false, retain=true 2024-11-11T22:26:56,304 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=1077b11658f2ec3dcbfd684d096e1030, regionState=OPENING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, REOPEN/MOVE because future has completed 2024-11-11T22:26:56,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:26:56,461 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,461 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:26:56,462 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,462 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:26:56,462 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,462 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,463 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,464 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1077b11658f2ec3dcbfd684d096e1030 columnFamilyName A 2024-11-11T22:26:56,465 DEBUG [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:56,465 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(327): Store=1077b11658f2ec3dcbfd684d096e1030/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:26:56,466 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,466 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1077b11658f2ec3dcbfd684d096e1030 columnFamilyName B 2024-11-11T22:26:56,466 DEBUG [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:56,467 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(327): Store=1077b11658f2ec3dcbfd684d096e1030/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:26:56,467 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,468 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1077b11658f2ec3dcbfd684d096e1030 columnFamilyName C 2024-11-11T22:26:56,468 DEBUG [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:56,468 INFO [StoreOpener-1077b11658f2ec3dcbfd684d096e1030-1 {}] regionserver.HStore(327): Store=1077b11658f2ec3dcbfd684d096e1030/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:26:56,468 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,469 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,469 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,470 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,470 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,470 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:26:56,471 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,472 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 1077b11658f2ec3dcbfd684d096e1030; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65720499, jitterRate=-0.0206882506608963}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:26:56,472 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,473 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 1077b11658f2ec3dcbfd684d096e1030: Running coprocessor pre-open hook at 1731364016462Writing region info on filesystem at 1731364016462Initializing all the Stores at 1731364016463 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364016463Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364016463Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364016463Cleaning up temporary data from old regions at 1731364016470 (+7 ms)Running coprocessor post-open hooks at 1731364016472 (+2 ms)Region opened successfully at 1731364016473 (+1 ms) 2024-11-11T22:26:56,474 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., pid=92, masterSystemTime=1731364016458 2024-11-11T22:26:56,476 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=1077b11658f2ec3dcbfd684d096e1030, regionState=OPEN, openSeqNum=5, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,476 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,476 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:26:56,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-11T22:26:56,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 in 172 msec 2024-11-11T22:26:56,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-11T22:26:56,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, REOPEN/MOVE in 494 msec 2024-11-11T22:26:56,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-11T22:26:56,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-11-11T22:26:56,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 919 msec 2024-11-11T22:26:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T22:26:56,487 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:26:56,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:56,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:56,488 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3406cbf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:56,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:56,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:56,489 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc548b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:56,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:56,491 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@724425dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:56,493 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1246f42b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:56,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:56,500 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d5dc702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:56,502 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4123805f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:56,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:56,504 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@336b7ec1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:56,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:56,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:56,505 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6036523f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:56,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:56,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b1dfe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:56,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:56,510 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550aff34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:56,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:56,513 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4edc5a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:56,515 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:56,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b210bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:56,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:56,517 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1067ed0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:56,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:56,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:56,518 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b126bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:56,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:56,521 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:56,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:56,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af0fc43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:56,523 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:56,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ed2520a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,524 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:56,524 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:56,525 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135739ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:56,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:56,528 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d760efb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:26:56,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:26:56,537 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:26:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:26:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:26:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68d14432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:26:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:26:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:26:56,539 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:26:56,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df4919e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:26:56,541 DEBUG [Thread-1528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:56,541 DEBUG [Thread-1539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:56,541 DEBUG [Thread-1531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:56,542 DEBUG [Thread-1533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:56,543 DEBUG [Thread-1533 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:56,543 DEBUG [Thread-1533 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:56,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:56,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:56,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:56,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:26:56,548 DEBUG [Thread-1535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:26:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:26:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:26:56,550 DEBUG [Thread-1535 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:56,551 DEBUG [Thread-1535 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:56,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:26:56,560 DEBUG [Thread-1520 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:56,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:26:56,564 DEBUG [Thread-1537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:56,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:56,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:26:56,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:56,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:26:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:56,579 DEBUG [Thread-1526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:56,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:26:56,589 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:26:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-11T22:26:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T22:26:56,592 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:26:56,593 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:26:56,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:26:56,595 DEBUG [Thread-1522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:56,602 DEBUG [Thread-1522 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:56,602 DEBUG [Thread-1522 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:56,605 DEBUG [Thread-1524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:26:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:26:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:26:56,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:56,608 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:26:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:26:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:26:56,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111154bb513ed63f476bb22db6971b5a6cc2_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_1/A:col10/1731364016576/Put/seqid=0 2024-11-11T22:26:56,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742158_1334 (size=9714) 2024-11-11T22:26:56,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364076645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364076650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364076651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364076657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364076657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T22:26:56,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T22:26:56,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:26:56,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:56,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:56,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364076753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364076763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364076763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364076763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364076763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T22:26:56,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:26:56,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:56,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:56,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T22:26:56,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364076963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364076975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364076975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364076975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364076977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,031 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:57,034 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111154bb513ed63f476bb22db6971b5a6cc2_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154bb513ed63f476bb22db6971b5a6cc2_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:57,035 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/ece7085753fc4ecf8d5f88e2d3a2e0c5, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:26:57,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/ece7085753fc4ecf8d5f88e2d3a2e0c5 is 175, key is test_row_1/A:col10/1731364016576/Put/seqid=0 2024-11-11T22:26:57,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742159_1335 (size=22359) 2024-11-11T22:26:57,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T22:26:57,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:26:57,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T22:26:57,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:26:57,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T22:26:57,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364077273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364077283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:57,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364077284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364077284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364077292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T22:26:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:26:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,448 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/ece7085753fc4ecf8d5f88e2d3a2e0c5 2024-11-11T22:26:57,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/4aa74d0ebeee4181b04d746cc16bf9b0 is 50, key is test_row_1/B:col10/1731364016576/Put/seqid=0 2024-11-11T22:26:57,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742160_1336 (size=7315) 2024-11-11T22:26:57,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T22:26:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:26:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T22:26:57,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:26:57,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T22:26:57,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364077785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364077793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:57,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364077793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364077793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364077808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:57,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T22:26:57,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:26:57,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/4aa74d0ebeee4181b04d746cc16bf9b0 2024-11-11T22:26:57,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/1106e8f6207a493bbaf5ec40e77479ae is 50, key is test_row_1/C:col10/1731364016576/Put/seqid=0 2024-11-11T22:26:57,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742161_1337 (size=7315) 2024-11-11T22:26:57,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/1106e8f6207a493bbaf5ec40e77479ae 2024-11-11T22:26:57,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/ece7085753fc4ecf8d5f88e2d3a2e0c5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/ece7085753fc4ecf8d5f88e2d3a2e0c5 2024-11-11T22:26:57,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/ece7085753fc4ecf8d5f88e2d3a2e0c5, entries=100, sequenceid=15, filesize=21.8 K 2024-11-11T22:26:57,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/4aa74d0ebeee4181b04d746cc16bf9b0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4aa74d0ebeee4181b04d746cc16bf9b0 2024-11-11T22:26:57,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4aa74d0ebeee4181b04d746cc16bf9b0, entries=50, sequenceid=15, filesize=7.1 K 2024-11-11T22:26:57,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/1106e8f6207a493bbaf5ec40e77479ae as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1106e8f6207a493bbaf5ec40e77479ae 2024-11-11T22:26:57,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T22:26:57,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:26:57,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:57,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:26:57,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1106e8f6207a493bbaf5ec40e77479ae, entries=50, sequenceid=15, filesize=7.1 K 2024-11-11T22:26:57,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1077b11658f2ec3dcbfd684d096e1030 in 1419ms, sequenceid=15, compaction requested=false 2024-11-11T22:26:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:26:58,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T22:26:58,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:58,143 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T22:26:58,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119385893e0b1d47d09a8f679c243c3ec1_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364016649/Put/seqid=0 2024-11-11T22:26:58,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742162_1338 (size=12154) 2024-11-11T22:26:58,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:58,161 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119385893e0b1d47d09a8f679c243c3ec1_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119385893e0b1d47d09a8f679c243c3ec1_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:58,161 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:26:58,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/1b8c3eb0a6e04c9196982bb23028a8b1, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:26:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/1b8c3eb0a6e04c9196982bb23028a8b1 is 175, key is test_row_0/A:col10/1731364016649/Put/seqid=0 2024-11-11T22:26:58,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742163_1339 (size=30955) 2024-11-11T22:26:58,194 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/1b8c3eb0a6e04c9196982bb23028a8b1 2024-11-11T22:26:58,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8bb6207d86694a7cb801a1829d0569b5 is 50, key is test_row_0/B:col10/1731364016649/Put/seqid=0 2024-11-11T22:26:58,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742164_1340 (size=12001) 2024-11-11T22:26:58,613 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8bb6207d86694a7cb801a1829d0569b5 2024-11-11T22:26:58,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/81c993ed7306439e81d65d472a605fd1 is 50, key is test_row_0/C:col10/1731364016649/Put/seqid=0 2024-11-11T22:26:58,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742165_1341 (size=12001) 2024-11-11T22:26:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T22:26:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:26:58,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:58,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364078816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364078817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:58,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364078818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:58,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364078821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:58,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:58,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364078823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:58,830 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:26:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:58,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364078933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:58,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364078933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:58,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364078933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364078933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,040 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/81c993ed7306439e81d65d472a605fd1 2024-11-11T22:26:59,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/1b8c3eb0a6e04c9196982bb23028a8b1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1b8c3eb0a6e04c9196982bb23028a8b1 2024-11-11T22:26:59,050 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1b8c3eb0a6e04c9196982bb23028a8b1, entries=150, sequenceid=41, filesize=30.2 K 2024-11-11T22:26:59,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8bb6207d86694a7cb801a1829d0569b5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8bb6207d86694a7cb801a1829d0569b5 2024-11-11T22:26:59,056 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8bb6207d86694a7cb801a1829d0569b5, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T22:26:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/81c993ed7306439e81d65d472a605fd1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/81c993ed7306439e81d65d472a605fd1 2024-11-11T22:26:59,074 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/81c993ed7306439e81d65d472a605fd1, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T22:26:59,075 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1077b11658f2ec3dcbfd684d096e1030 in 932ms, sequenceid=41, compaction requested=false 2024-11-11T22:26:59,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:26:59,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:59,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-11T22:26:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-11T22:26:59,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-11T22:26:59,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4840 sec 2024-11-11T22:26:59,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.4900 sec 2024-11-11T22:26:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:59,148 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:59,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:26:59,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d037afbd00b74c5c99f2b33944684476_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364019148/Put/seqid=0 2024-11-11T22:26:59,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742166_1342 (size=17034) 2024-11-11T22:26:59,172 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:59,182 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d037afbd00b74c5c99f2b33944684476_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d037afbd00b74c5c99f2b33944684476_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:59,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/5179bef741304c7e9c8904bbcb251828, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:26:59,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/5179bef741304c7e9c8904bbcb251828 is 175, key is test_row_0/A:col10/1731364019148/Put/seqid=0 2024-11-11T22:26:59,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742167_1343 (size=48139) 2024-11-11T22:26:59,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364079190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364079197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364079202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364079203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364079313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364079314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364079314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364079323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364079523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364079524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364079524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364079533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,594 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/5179bef741304c7e9c8904bbcb251828 2024-11-11T22:26:59,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/894705c39dc3434a80bf2c3143e59e40 is 50, key is test_row_0/B:col10/1731364019148/Put/seqid=0 2024-11-11T22:26:59,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742168_1344 (size=12001) 2024-11-11T22:26:59,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/894705c39dc3434a80bf2c3143e59e40 2024-11-11T22:26:59,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/f789c26f6157442989a1007e3685bbe2 is 50, key is test_row_0/C:col10/1731364019148/Put/seqid=0 2024-11-11T22:26:59,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742169_1345 (size=12001) 2024-11-11T22:26:59,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/f789c26f6157442989a1007e3685bbe2 2024-11-11T22:26:59,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/5179bef741304c7e9c8904bbcb251828 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5179bef741304c7e9c8904bbcb251828 2024-11-11T22:26:59,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5179bef741304c7e9c8904bbcb251828, entries=250, sequenceid=55, filesize=47.0 K 2024-11-11T22:26:59,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/894705c39dc3434a80bf2c3143e59e40 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/894705c39dc3434a80bf2c3143e59e40 2024-11-11T22:26:59,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/894705c39dc3434a80bf2c3143e59e40, entries=150, sequenceid=55, filesize=11.7 K 2024-11-11T22:26:59,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/f789c26f6157442989a1007e3685bbe2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/f789c26f6157442989a1007e3685bbe2 2024-11-11T22:26:59,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/f789c26f6157442989a1007e3685bbe2, entries=150, sequenceid=55, filesize=11.7 K 2024-11-11T22:26:59,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1077b11658f2ec3dcbfd684d096e1030 in 512ms, sequenceid=55, compaction requested=true 2024-11-11T22:26:59,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:26:59,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:26:59,660 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:59,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:59,660 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:26:59,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:26:59,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:26:59,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:26:59,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:26:59,661 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:59,662 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/B is initiating minor compaction (all files) 2024-11-11T22:26:59,662 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/B in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:59,662 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4aa74d0ebeee4181b04d746cc16bf9b0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8bb6207d86694a7cb801a1829d0569b5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/894705c39dc3434a80bf2c3143e59e40] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=30.6 K 2024-11-11T22:26:59,662 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101453 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:26:59,662 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/A is initiating minor compaction (all files) 2024-11-11T22:26:59,662 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/A in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:59,662 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/ece7085753fc4ecf8d5f88e2d3a2e0c5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1b8c3eb0a6e04c9196982bb23028a8b1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5179bef741304c7e9c8904bbcb251828] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=99.1 K 2024-11-11T22:26:59,662 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:26:59,662 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/ece7085753fc4ecf8d5f88e2d3a2e0c5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1b8c3eb0a6e04c9196982bb23028a8b1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5179bef741304c7e9c8904bbcb251828] 2024-11-11T22:26:59,663 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa74d0ebeee4181b04d746cc16bf9b0, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731364016576 2024-11-11T22:26:59,668 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting ece7085753fc4ecf8d5f88e2d3a2e0c5, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731364016574 2024-11-11T22:26:59,674 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bb6207d86694a7cb801a1829d0569b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731364016641 2024-11-11T22:26:59,674 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b8c3eb0a6e04c9196982bb23028a8b1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731364016641 2024-11-11T22:26:59,681 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 894705c39dc3434a80bf2c3143e59e40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731364018815 2024-11-11T22:26:59,683 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5179bef741304c7e9c8904bbcb251828, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731364018815 2024-11-11T22:26:59,689 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:26:59,690 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/99a5c3d80fc04b14b94b99985e383460 is 50, key is test_row_0/B:col10/1731364019148/Put/seqid=0 2024-11-11T22:26:59,691 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:26:59,697 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111150291ebc61e547bfaab5c84578c3305b_1077b11658f2ec3dcbfd684d096e1030 store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:26:59,699 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111150291ebc61e547bfaab5c84578c3305b_1077b11658f2ec3dcbfd684d096e1030, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:26:59,699 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111150291ebc61e547bfaab5c84578c3305b_1077b11658f2ec3dcbfd684d096e1030 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:26:59,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742170_1346 (size=12104) 2024-11-11T22:26:59,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742171_1347 (size=4469) 2024-11-11T22:26:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:59,837 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:26:59,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:26:59,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111faa7da295b94d93932ad776e2af17e4_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364019202/Put/seqid=0 2024-11-11T22:26:59,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364079860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742172_1348 (size=17034) 2024-11-11T22:26:59,870 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:26:59,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111faa7da295b94d93932ad776e2af17e4_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111faa7da295b94d93932ad776e2af17e4_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:26:59,875 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/39524dfbd9ce465f83bd9e391950f296, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:26:59,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364079862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364079862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364079863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/39524dfbd9ce465f83bd9e391950f296 is 175, key is test_row_0/A:col10/1731364019202/Put/seqid=0 2024-11-11T22:26:59,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742173_1349 (size=48139) 2024-11-11T22:26:59,909 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/39524dfbd9ce465f83bd9e391950f296 2024-11-11T22:26:59,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/bad7e3f37f5d40b38dc972ac2f49244e is 50, key is test_row_0/B:col10/1731364019202/Put/seqid=0 2024-11-11T22:26:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742174_1350 (size=12001) 2024-11-11T22:26:59,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364079974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:26:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:26:59,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364079983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364079983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:26:59,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:26:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364079983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,110 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#A#compaction#298 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:00,110 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/3014c823c1a049e38f47dd492820a283 is 175, key is test_row_0/A:col10/1731364019148/Put/seqid=0 2024-11-11T22:27:00,112 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/99a5c3d80fc04b14b94b99985e383460 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/99a5c3d80fc04b14b94b99985e383460 2024-11-11T22:27:00,119 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/B of 1077b11658f2ec3dcbfd684d096e1030 into 99a5c3d80fc04b14b94b99985e383460(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:00,119 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:00,119 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/B, priority=13, startTime=1731364019660; duration=0sec 2024-11-11T22:27:00,119 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:00,120 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:B 2024-11-11T22:27:00,120 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:00,120 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:00,121 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/C is initiating minor compaction (all files) 2024-11-11T22:27:00,121 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/C in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:00,121 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1106e8f6207a493bbaf5ec40e77479ae, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/81c993ed7306439e81d65d472a605fd1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/f789c26f6157442989a1007e3685bbe2] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=30.6 K 2024-11-11T22:27:00,121 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 1106e8f6207a493bbaf5ec40e77479ae, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731364016576 2024-11-11T22:27:00,121 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 81c993ed7306439e81d65d472a605fd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731364016641 2024-11-11T22:27:00,122 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f789c26f6157442989a1007e3685bbe2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731364018815 2024-11-11T22:27:00,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742175_1351 (size=31058) 2024-11-11T22:27:00,129 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#C#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:00,129 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/a474eb13ccd941ceaa6120b2753e7636 is 50, key is test_row_0/C:col10/1731364019148/Put/seqid=0 2024-11-11T22:27:00,137 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/3014c823c1a049e38f47dd492820a283 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3014c823c1a049e38f47dd492820a283 2024-11-11T22:27:00,144 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/A of 1077b11658f2ec3dcbfd684d096e1030 into 3014c823c1a049e38f47dd492820a283(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:00,144 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:00,144 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/A, priority=13, startTime=1731364019660; duration=0sec 2024-11-11T22:27:00,145 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:00,145 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:A 2024-11-11T22:27:00,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742176_1352 (size=12104) 2024-11-11T22:27:00,149 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/a474eb13ccd941ceaa6120b2753e7636 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a474eb13ccd941ceaa6120b2753e7636 2024-11-11T22:27:00,153 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/C of 1077b11658f2ec3dcbfd684d096e1030 into a474eb13ccd941ceaa6120b2753e7636(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:00,153 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:00,153 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/C, priority=13, startTime=1731364019660; duration=0sec 2024-11-11T22:27:00,153 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:00,153 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:C 2024-11-11T22:27:00,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364080183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364080213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:00,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364080213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364080213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/bad7e3f37f5d40b38dc972ac2f49244e 2024-11-11T22:27:00,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/a13b7c0ada2a42f5b4c7b9e3a16f4c26 is 50, key is test_row_0/C:col10/1731364019202/Put/seqid=0 2024-11-11T22:27:00,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742177_1353 (size=12001) 2024-11-11T22:27:00,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364080493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364080523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364080523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364080532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:00,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/a13b7c0ada2a42f5b4c7b9e3a16f4c26 2024-11-11T22:27:00,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/39524dfbd9ce465f83bd9e391950f296 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/39524dfbd9ce465f83bd9e391950f296 2024-11-11T22:27:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T22:27:00,788 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:00,789 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-11T22:27:00,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/39524dfbd9ce465f83bd9e391950f296, entries=250, sequenceid=79, filesize=47.0 K 2024-11-11T22:27:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:27:00,792 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:00,794 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:00,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:00,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/bad7e3f37f5d40b38dc972ac2f49244e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/bad7e3f37f5d40b38dc972ac2f49244e 2024-11-11T22:27:00,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/bad7e3f37f5d40b38dc972ac2f49244e, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T22:27:00,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/a13b7c0ada2a42f5b4c7b9e3a16f4c26 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a13b7c0ada2a42f5b4c7b9e3a16f4c26 2024-11-11T22:27:00,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a13b7c0ada2a42f5b4c7b9e3a16f4c26, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T22:27:00,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1077b11658f2ec3dcbfd684d096e1030 in 970ms, sequenceid=79, compaction requested=false 2024-11-11T22:27:00,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:00,846 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:00,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T22:27:00,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dc78278c06804fd398e856208690d384_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364020844/Put/seqid=0 2024-11-11T22:27:00,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742178_1354 (size=14594) 2024-11-11T22:27:00,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:00,875 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dc78278c06804fd398e856208690d384_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dc78278c06804fd398e856208690d384_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:00,876 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/2807f3fe37ea43fa820ebc68cb7eb3ef, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:00,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/2807f3fe37ea43fa820ebc68cb7eb3ef is 175, key is test_row_0/A:col10/1731364020844/Put/seqid=0 2024-11-11T22:27:00,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742179_1355 (size=39549) 2024-11-11T22:27:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:27:00,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:27:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:00,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:00,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364081001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:01,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364081003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:01,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364081043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364081043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364081043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:01,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:27:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:01,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:27:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364081115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:01,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:27:01,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:01,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/2807f3fe37ea43fa820ebc68cb7eb3ef 2024-11-11T22:27:01,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/5f59e896a08c4da1b80cac09c6d41fea is 50, key is test_row_0/B:col10/1731364020844/Put/seqid=0 2024-11-11T22:27:01,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742180_1356 (size=12001) 2024-11-11T22:27:01,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364081323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:01,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:27:01,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:01,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:27:01,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:27:01,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:01,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364081643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:01,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/5f59e896a08c4da1b80cac09c6d41fea 2024-11-11T22:27:01,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:27:01,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:01,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/27d1884487ca4c3c8b7400a21a238eba is 50, key is test_row_0/C:col10/1731364020844/Put/seqid=0 2024-11-11T22:27:01,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742181_1357 (size=12001) 2024-11-11T22:27:01,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:27:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:01,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:27:02,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:27:02,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:02,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:02,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364082023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:02,030 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:02,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:02,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:02,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364082064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:02,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364082064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:02,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364082064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:02,071 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:02,072 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:02,072 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:02,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/27d1884487ca4c3c8b7400a21a238eba 2024-11-11T22:27:02,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/2807f3fe37ea43fa820ebc68cb7eb3ef as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/2807f3fe37ea43fa820ebc68cb7eb3ef 2024-11-11T22:27:02,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/2807f3fe37ea43fa820ebc68cb7eb3ef, entries=200, sequenceid=95, filesize=38.6 K 2024-11-11T22:27:02,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/5f59e896a08c4da1b80cac09c6d41fea as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/5f59e896a08c4da1b80cac09c6d41fea 2024-11-11T22:27:02,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/5f59e896a08c4da1b80cac09c6d41fea, entries=150, sequenceid=95, filesize=11.7 K 2024-11-11T22:27:02,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/27d1884487ca4c3c8b7400a21a238eba as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/27d1884487ca4c3c8b7400a21a238eba 2024-11-11T22:27:02,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364082154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:02,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/27d1884487ca4c3c8b7400a21a238eba, entries=150, sequenceid=95, filesize=11.7 K 2024-11-11T22:27:02,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1077b11658f2ec3dcbfd684d096e1030 in 1320ms, sequenceid=95, compaction requested=true 2024-11-11T22:27:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:02,166 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:02,166 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:02,167 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118746 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:02,167 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/A is initiating minor compaction (all files) 2024-11-11T22:27:02,167 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:02,167 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/A in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:02,167 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/B is initiating minor compaction (all files) 2024-11-11T22:27:02,167 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/B in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:02,167 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3014c823c1a049e38f47dd492820a283, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/39524dfbd9ce465f83bd9e391950f296, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/2807f3fe37ea43fa820ebc68cb7eb3ef] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=116.0 K 2024-11-11T22:27:02,167 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/99a5c3d80fc04b14b94b99985e383460, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/bad7e3f37f5d40b38dc972ac2f49244e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/5f59e896a08c4da1b80cac09c6d41fea] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=35.3 K 2024-11-11T22:27:02,167 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:02,167 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3014c823c1a049e38f47dd492820a283, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/39524dfbd9ce465f83bd9e391950f296, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/2807f3fe37ea43fa820ebc68cb7eb3ef] 2024-11-11T22:27:02,167 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3014c823c1a049e38f47dd492820a283, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731364018815 2024-11-11T22:27:02,167 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 99a5c3d80fc04b14b94b99985e383460, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731364018815 2024-11-11T22:27:02,168 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39524dfbd9ce465f83bd9e391950f296, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731364019187 2024-11-11T22:27:02,168 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting bad7e3f37f5d40b38dc972ac2f49244e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731364019187 2024-11-11T22:27:02,168 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2807f3fe37ea43fa820ebc68cb7eb3ef, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731364019858 2024-11-11T22:27:02,168 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f59e896a08c4da1b80cac09c6d41fea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731364019858 2024-11-11T22:27:02,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:27:02,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:02,170 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:27:02,181 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:02,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111198f4a7e497d843e5a85219f78b165905_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364020999/Put/seqid=0 2024-11-11T22:27:02,199 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118cdfc6f37af947268bb573bbf11a8931_1077b11658f2ec3dcbfd684d096e1030 store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:02,201 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#B#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:02,201 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118cdfc6f37af947268bb573bbf11a8931_1077b11658f2ec3dcbfd684d096e1030, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:02,201 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118cdfc6f37af947268bb573bbf11a8931_1077b11658f2ec3dcbfd684d096e1030 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:02,201 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/b3689f75fb9e41f082a64a2d1adf2bfe is 50, key is test_row_0/B:col10/1731364020844/Put/seqid=0 2024-11-11T22:27:02,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742182_1358 (size=12154) 2024-11-11T22:27:02,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:02,210 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111198f4a7e497d843e5a85219f78b165905_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111198f4a7e497d843e5a85219f78b165905_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:02,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/cd9810cba29342a3a12a41f428ca0d65, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:02,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/cd9810cba29342a3a12a41f428ca0d65 is 175, key is test_row_0/A:col10/1731364020999/Put/seqid=0 2024-11-11T22:27:02,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742185_1361 (size=30955) 2024-11-11T22:27:02,225 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/cd9810cba29342a3a12a41f428ca0d65 2024-11-11T22:27:02,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742183_1359 (size=4469) 2024-11-11T22:27:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/d98384b9fbd744d5a4c81de5a7b9a09c is 50, key is test_row_0/B:col10/1731364020999/Put/seqid=0 2024-11-11T22:27:02,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742184_1360 (size=12207) 2024-11-11T22:27:02,249 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/b3689f75fb9e41f082a64a2d1adf2bfe as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/b3689f75fb9e41f082a64a2d1adf2bfe 2024-11-11T22:27:02,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742186_1362 (size=12001) 2024-11-11T22:27:02,255 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/B of 1077b11658f2ec3dcbfd684d096e1030 into b3689f75fb9e41f082a64a2d1adf2bfe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:02,255 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:02,255 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/B, priority=13, startTime=1731364022166; duration=0sec 2024-11-11T22:27:02,255 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:02,255 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:B 2024-11-11T22:27:02,256 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:02,256 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:02,256 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/C is initiating minor compaction (all files) 2024-11-11T22:27:02,257 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/C in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:02,257 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a474eb13ccd941ceaa6120b2753e7636, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a13b7c0ada2a42f5b4c7b9e3a16f4c26, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/27d1884487ca4c3c8b7400a21a238eba] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=35.3 K 2024-11-11T22:27:02,257 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a474eb13ccd941ceaa6120b2753e7636, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731364018815 2024-11-11T22:27:02,257 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a13b7c0ada2a42f5b4c7b9e3a16f4c26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731364019187 2024-11-11T22:27:02,258 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 27d1884487ca4c3c8b7400a21a238eba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731364019858 2024-11-11T22:27:02,269 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#C#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:02,270 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/0c3226119686430786ea722857f3c180 is 50, key is test_row_0/C:col10/1731364020844/Put/seqid=0 2024-11-11T22:27:02,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742187_1363 (size=12207) 2024-11-11T22:27:02,635 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#A#compaction#306 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:02,636 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/1a9b42b97b7f4b1690da12698a3995de is 175, key is test_row_0/A:col10/1731364020844/Put/seqid=0 2024-11-11T22:27:02,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742188_1364 (size=31161) 2024-11-11T22:27:02,648 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/1a9b42b97b7f4b1690da12698a3995de as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1a9b42b97b7f4b1690da12698a3995de 2024-11-11T22:27:02,651 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/d98384b9fbd744d5a4c81de5a7b9a09c 2024-11-11T22:27:02,653 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/A of 1077b11658f2ec3dcbfd684d096e1030 into 1a9b42b97b7f4b1690da12698a3995de(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:02,653 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:02,653 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/A, priority=13, startTime=1731364022166; duration=0sec 2024-11-11T22:27:02,653 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:02,653 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:A 2024-11-11T22:27:02,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/6a2c3035b725495d9578475a930f2e78 is 50, key is test_row_0/C:col10/1731364020999/Put/seqid=0 2024-11-11T22:27:02,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742189_1365 (size=12001) 2024-11-11T22:27:02,687 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/0c3226119686430786ea722857f3c180 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/0c3226119686430786ea722857f3c180 2024-11-11T22:27:02,695 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/C of 1077b11658f2ec3dcbfd684d096e1030 into 0c3226119686430786ea722857f3c180(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:02,695 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:02,695 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/C, priority=13, startTime=1731364022166; duration=0sec 2024-11-11T22:27:02,695 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:02,695 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:C 2024-11-11T22:27:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:27:03,073 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/6a2c3035b725495d9578475a930f2e78 2024-11-11T22:27:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/cd9810cba29342a3a12a41f428ca0d65 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/cd9810cba29342a3a12a41f428ca0d65 2024-11-11T22:27:03,081 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/cd9810cba29342a3a12a41f428ca0d65, entries=150, sequenceid=118, filesize=30.2 K 2024-11-11T22:27:03,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/d98384b9fbd744d5a4c81de5a7b9a09c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d98384b9fbd744d5a4c81de5a7b9a09c 2024-11-11T22:27:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,086 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d98384b9fbd744d5a4c81de5a7b9a09c, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T22:27:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/6a2c3035b725495d9578475a930f2e78 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/6a2c3035b725495d9578475a930f2e78 2024-11-11T22:27:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,097 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/6a2c3035b725495d9578475a930f2e78, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T22:27:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,097 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 1077b11658f2ec3dcbfd684d096e1030 in 927ms, sequenceid=118, compaction requested=false 2024-11-11T22:27:03,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:03,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:03,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-11T22:27:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-11T22:27:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T22:27:03,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3050 sec 2024-11-11T22:27:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.3130 sec 2024-11-11T22:27:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:03,387 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:03,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:27:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ead40c3e95aa49f3a9e052261670af55_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364023315/Put/seqid=0 2024-11-11T22:27:03,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742190_1366 (size=17234) 2024-11-11T22:27:03,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:03,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ead40c3e95aa49f3a9e052261670af55_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ead40c3e95aa49f3a9e052261670af55_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:03,468 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/31e3b9849d32461b8ffd7c2833bf4609, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:03,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/31e3b9849d32461b8ffd7c2833bf4609 is 175, key is test_row_0/A:col10/1731364023315/Put/seqid=0 2024-11-11T22:27:03,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742191_1367 (size=48339) 2024-11-11T22:27:03,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364083593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:03,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364083713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:03,874 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/31e3b9849d32461b8ffd7c2833bf4609 2024-11-11T22:27:03,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/6b27b8ac858d47d6888aaa5a7c6c69da is 50, key is test_row_0/B:col10/1731364023315/Put/seqid=0 2024-11-11T22:27:03,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742192_1368 (size=12101) 2024-11-11T22:27:03,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364083933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:04,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364084054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:04,063 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:04,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364084084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:04,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364084084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:04,091 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:04,091 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:04,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364084093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:04,104 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:04,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:04,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364084253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:04,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/6b27b8ac858d47d6888aaa5a7c6c69da 2024-11-11T22:27:04,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/c577cab129564f0abf9b51318b9ead0d is 50, key is test_row_0/C:col10/1731364023315/Put/seqid=0 2024-11-11T22:27:04,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742193_1369 (size=12101) 2024-11-11T22:27:04,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/c577cab129564f0abf9b51318b9ead0d 2024-11-11T22:27:04,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/31e3b9849d32461b8ffd7c2833bf4609 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/31e3b9849d32461b8ffd7c2833bf4609 2024-11-11T22:27:04,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/31e3b9849d32461b8ffd7c2833bf4609, entries=250, sequenceid=132, filesize=47.2 K 2024-11-11T22:27:04,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/6b27b8ac858d47d6888aaa5a7c6c69da as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/6b27b8ac858d47d6888aaa5a7c6c69da 2024-11-11T22:27:04,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/6b27b8ac858d47d6888aaa5a7c6c69da, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T22:27:04,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/c577cab129564f0abf9b51318b9ead0d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/c577cab129564f0abf9b51318b9ead0d 2024-11-11T22:27:04,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/c577cab129564f0abf9b51318b9ead0d, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T22:27:04,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1077b11658f2ec3dcbfd684d096e1030 in 958ms, sequenceid=132, compaction requested=true 2024-11-11T22:27:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:04,345 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:04,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:04,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:04,346 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:04,346 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:04,346 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/A is initiating minor compaction (all files) 2024-11-11T22:27:04,347 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/A in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:04,347 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1a9b42b97b7f4b1690da12698a3995de, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/cd9810cba29342a3a12a41f428ca0d65, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/31e3b9849d32461b8ffd7c2833bf4609] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=107.9 K 2024-11-11T22:27:04,347 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:04,347 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1a9b42b97b7f4b1690da12698a3995de, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/cd9810cba29342a3a12a41f428ca0d65, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/31e3b9849d32461b8ffd7c2833bf4609] 2024-11-11T22:27:04,347 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:04,347 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/B is initiating minor compaction (all files) 2024-11-11T22:27:04,347 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/B in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:04,347 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/b3689f75fb9e41f082a64a2d1adf2bfe, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d98384b9fbd744d5a4c81de5a7b9a09c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/6b27b8ac858d47d6888aaa5a7c6c69da] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=35.5 K 2024-11-11T22:27:04,347 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting b3689f75fb9e41f082a64a2d1adf2bfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731364019858 2024-11-11T22:27:04,347 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a9b42b97b7f4b1690da12698a3995de, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731364019858 2024-11-11T22:27:04,347 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting d98384b9fbd744d5a4c81de5a7b9a09c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731364020968 2024-11-11T22:27:04,348 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd9810cba29342a3a12a41f428ca0d65, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731364020968 2024-11-11T22:27:04,348 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b27b8ac858d47d6888aaa5a7c6c69da, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731364023315 2024-11-11T22:27:04,348 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31e3b9849d32461b8ffd7c2833bf4609, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731364023315 2024-11-11T22:27:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:04,357 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:04,357 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8e5ccc5798134eef988a8cd5aacdfea2 is 50, key is test_row_0/B:col10/1731364023315/Put/seqid=0 2024-11-11T22:27:04,370 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:04,378 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111150b772ec92604ada80087ab11867cc0c_1077b11658f2ec3dcbfd684d096e1030 store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:04,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742194_1370 (size=12409) 2024-11-11T22:27:04,380 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111150b772ec92604ada80087ab11867cc0c_1077b11658f2ec3dcbfd684d096e1030, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:04,380 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111150b772ec92604ada80087ab11867cc0c_1077b11658f2ec3dcbfd684d096e1030 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:04,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742195_1371 (size=4469) 2024-11-11T22:27:04,391 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8e5ccc5798134eef988a8cd5aacdfea2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8e5ccc5798134eef988a8cd5aacdfea2 2024-11-11T22:27:04,399 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/B of 1077b11658f2ec3dcbfd684d096e1030 into 8e5ccc5798134eef988a8cd5aacdfea2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:04,399 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:04,399 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/B, priority=13, startTime=1731364024346; duration=0sec 2024-11-11T22:27:04,399 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:04,400 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:B 2024-11-11T22:27:04,400 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:04,401 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:04,401 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/C is initiating minor compaction (all files) 2024-11-11T22:27:04,401 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/C in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:04,401 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/0c3226119686430786ea722857f3c180, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/6a2c3035b725495d9578475a930f2e78, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/c577cab129564f0abf9b51318b9ead0d] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=35.5 K 2024-11-11T22:27:04,402 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c3226119686430786ea722857f3c180, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731364019858 2024-11-11T22:27:04,402 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2c3035b725495d9578475a930f2e78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731364020968 2024-11-11T22:27:04,403 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting c577cab129564f0abf9b51318b9ead0d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731364023315 2024-11-11T22:27:04,416 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:04,417 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/4cfe29a42a304507a3810ab956438119 is 50, key is test_row_0/C:col10/1731364023315/Put/seqid=0 2024-11-11T22:27:04,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742196_1372 (size=12409) 2024-11-11T22:27:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:04,776 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:04,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T22:27:04,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111bc16c7af03348b9a087d97276e0e759_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364023591/Put/seqid=0 2024-11-11T22:27:04,790 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#A#compaction#316 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:04,791 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/3ef3d986844d4a199df0a55d5b5fae4c is 175, key is test_row_0/A:col10/1731364023315/Put/seqid=0 2024-11-11T22:27:04,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742197_1373 (size=14794) 2024-11-11T22:27:04,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742198_1374 (size=31363) 2024-11-11T22:27:04,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364084822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:04,840 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/4cfe29a42a304507a3810ab956438119 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/4cfe29a42a304507a3810ab956438119 2024-11-11T22:27:04,844 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/C of 1077b11658f2ec3dcbfd684d096e1030 into 4cfe29a42a304507a3810ab956438119(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:04,844 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:04,844 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/C, priority=13, startTime=1731364024349; duration=0sec 2024-11-11T22:27:04,844 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:04,845 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:C 2024-11-11T22:27:04,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364084943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:04,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:04,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:27:04,973 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:04,974 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-11T22:27:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:27:04,975 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:04,976 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:04,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:27:05,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:27:05,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:05,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364085153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:05,212 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:05,218 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/3ef3d986844d4a199df0a55d5b5fae4c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3ef3d986844d4a199df0a55d5b5fae4c 2024-11-11T22:27:05,220 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111bc16c7af03348b9a087d97276e0e759_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111bc16c7af03348b9a087d97276e0e759_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:05,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/fef0ad7e17c04aaeabb3d19654283afc, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:05,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/fef0ad7e17c04aaeabb3d19654283afc is 175, key is test_row_0/A:col10/1731364023591/Put/seqid=0 2024-11-11T22:27:05,223 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/A of 1077b11658f2ec3dcbfd684d096e1030 into 3ef3d986844d4a199df0a55d5b5fae4c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:05,223 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:05,223 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/A, priority=13, startTime=1731364024345; duration=0sec 2024-11-11T22:27:05,223 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:05,223 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:A 2024-11-11T22:27:05,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742199_1375 (size=39749) 2024-11-11T22:27:05,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:27:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:27:05,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:27:05,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:05,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364085463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:05,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:27:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:27:05,646 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/fef0ad7e17c04aaeabb3d19654283afc 2024-11-11T22:27:05,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/74a60f4cfd134915a9b76f41bd9a96cc is 50, key is test_row_0/B:col10/1731364023591/Put/seqid=0 2024-11-11T22:27:05,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742200_1376 (size=12151) 2024-11-11T22:27:05,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/74a60f4cfd134915a9b76f41bd9a96cc 2024-11-11T22:27:05,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/44f7e7d5f4cb4ee9a9f9b87bf8e69cbb is 50, key is test_row_0/C:col10/1731364023591/Put/seqid=0 2024-11-11T22:27:05,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742201_1377 (size=12151) 2024-11-11T22:27:05,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:27:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:05,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:27:05,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:05,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:05,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:05,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364085983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:06,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:27:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:06,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:06,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/44f7e7d5f4cb4ee9a9f9b87bf8e69cbb 2024-11-11T22:27:06,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/fef0ad7e17c04aaeabb3d19654283afc as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/fef0ad7e17c04aaeabb3d19654283afc 2024-11-11T22:27:06,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/fef0ad7e17c04aaeabb3d19654283afc, entries=200, sequenceid=159, filesize=38.8 K 2024-11-11T22:27:06,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/74a60f4cfd134915a9b76f41bd9a96cc as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/74a60f4cfd134915a9b76f41bd9a96cc 2024-11-11T22:27:06,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/74a60f4cfd134915a9b76f41bd9a96cc, entries=150, sequenceid=159, filesize=11.9 K 2024-11-11T22:27:06,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/44f7e7d5f4cb4ee9a9f9b87bf8e69cbb as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/44f7e7d5f4cb4ee9a9f9b87bf8e69cbb 2024-11-11T22:27:06,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/44f7e7d5f4cb4ee9a9f9b87bf8e69cbb, entries=150, sequenceid=159, filesize=11.9 K 2024-11-11T22:27:06,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1077b11658f2ec3dcbfd684d096e1030 in 1329ms, sequenceid=159, compaction requested=false 2024-11-11T22:27:06,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:27:06,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:27:06,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:06,200 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-11T22:27:06,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111112c9c82129854718b28778ca9a060692_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364024820/Put/seqid=0 2024-11-11T22:27:06,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742202_1378 (size=12304) 2024-11-11T22:27:06,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:06,623 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111112c9c82129854718b28778ca9a060692_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111112c9c82129854718b28778ca9a060692_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/d708e8110b884fd9826a2792f1bbf5b4, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/d708e8110b884fd9826a2792f1bbf5b4 is 175, key is test_row_0/A:col10/1731364024820/Put/seqid=0 2024-11-11T22:27:06,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742203_1379 (size=31105) 2024-11-11T22:27:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:07,034 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/d708e8110b884fd9826a2792f1bbf5b4 2024-11-11T22:27:07,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/50c4efb8ff7d44e3a280d7c21ebcd7b0 is 50, key is test_row_0/B:col10/1731364024820/Put/seqid=0 2024-11-11T22:27:07,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742204_1380 (size=12151) 2024-11-11T22:27:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:27:07,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364087132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:07,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:07,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:07,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:07,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364087243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:07,455 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/50c4efb8ff7d44e3a280d7c21ebcd7b0 2024-11-11T22:27:07,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364087453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:07,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/aaaf817bdd704e01b47d3399d028de9c is 50, key is test_row_0/C:col10/1731364024820/Put/seqid=0 2024-11-11T22:27:07,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742205_1381 (size=12151) 2024-11-11T22:27:07,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364087763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:07,872 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/aaaf817bdd704e01b47d3399d028de9c 2024-11-11T22:27:07,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/d708e8110b884fd9826a2792f1bbf5b4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/d708e8110b884fd9826a2792f1bbf5b4 2024-11-11T22:27:07,880 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/d708e8110b884fd9826a2792f1bbf5b4, entries=150, sequenceid=171, filesize=30.4 K 2024-11-11T22:27:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/50c4efb8ff7d44e3a280d7c21ebcd7b0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/50c4efb8ff7d44e3a280d7c21ebcd7b0 2024-11-11T22:27:07,884 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/50c4efb8ff7d44e3a280d7c21ebcd7b0, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T22:27:07,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/aaaf817bdd704e01b47d3399d028de9c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/aaaf817bdd704e01b47d3399d028de9c 2024-11-11T22:27:07,887 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/aaaf817bdd704e01b47d3399d028de9c, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T22:27:07,889 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 1077b11658f2ec3dcbfd684d096e1030 in 1690ms, sequenceid=171, compaction requested=true 2024-11-11T22:27:07,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:07,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:07,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T22:27:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-11T22:27:07,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T22:27:07,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9150 sec 2024-11-11T22:27:07,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.9190 sec 2024-11-11T22:27:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:08,078 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:08,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T22:27:08,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114eba8ae9831f40d880cc178865b9e4f6_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364027119/Put/seqid=0 2024-11-11T22:27:08,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742206_1382 (size=14794) 2024-11-11T22:27:08,095 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:08,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114eba8ae9831f40d880cc178865b9e4f6_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114eba8ae9831f40d880cc178865b9e4f6_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:08,099 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/8f364889cf3f4a10816c32ddc4464b79, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:08,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/8f364889cf3f4a10816c32ddc4464b79 is 175, key is test_row_0/A:col10/1731364027119/Put/seqid=0 2024-11-11T22:27:08,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742207_1383 (size=39749) 2024-11-11T22:27:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364088123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36082 deadline: 1731364088123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:08,124 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:08,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36076 deadline: 1731364088133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:08,140 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:08,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36022 deadline: 1731364088143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:08,151 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_0' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8289 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364088233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:08,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364088283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:08,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364088453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:08,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:08,508 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/8f364889cf3f4a10816c32ddc4464b79 2024-11-11T22:27:08,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/a9dd60fc761b4988b9e679ad539b470c is 50, key is test_row_0/B:col10/1731364027119/Put/seqid=0 2024-11-11T22:27:08,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742208_1384 (size=12151) 2024-11-11T22:27:08,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/a9dd60fc761b4988b9e679ad539b470c 2024-11-11T22:27:08,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/5e12258543f24da391a1d423bd7090a7 is 50, key is test_row_0/C:col10/1731364027119/Put/seqid=0 2024-11-11T22:27:08,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742209_1385 (size=12151) 2024-11-11T22:27:08,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/5e12258543f24da391a1d423bd7090a7 2024-11-11T22:27:08,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/8f364889cf3f4a10816c32ddc4464b79 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8f364889cf3f4a10816c32ddc4464b79 2024-11-11T22:27:08,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8f364889cf3f4a10816c32ddc4464b79, entries=200, sequenceid=198, filesize=38.8 K 2024-11-11T22:27:08,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/a9dd60fc761b4988b9e679ad539b470c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/a9dd60fc761b4988b9e679ad539b470c 2024-11-11T22:27:08,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/a9dd60fc761b4988b9e679ad539b470c, entries=150, sequenceid=198, filesize=11.9 K 2024-11-11T22:27:08,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/5e12258543f24da391a1d423bd7090a7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/5e12258543f24da391a1d423bd7090a7 2024-11-11T22:27:08,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/5e12258543f24da391a1d423bd7090a7, entries=150, sequenceid=198, filesize=11.9 K 2024-11-11T22:27:08,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1077b11658f2ec3dcbfd684d096e1030 in 483ms, sequenceid=198, compaction requested=true 2024-11-11T22:27:08,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:08,561 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:08,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:08,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:08,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:08,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:08,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:08,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:27:08,562 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:08,563 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:08,563 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:08,563 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/B is initiating minor compaction (all files) 2024-11-11T22:27:08,563 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/A is initiating minor compaction (all files) 2024-11-11T22:27:08,563 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/A in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:08,563 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/B in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:08,563 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3ef3d986844d4a199df0a55d5b5fae4c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/fef0ad7e17c04aaeabb3d19654283afc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/d708e8110b884fd9826a2792f1bbf5b4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8f364889cf3f4a10816c32ddc4464b79] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=138.6 K 2024-11-11T22:27:08,563 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8e5ccc5798134eef988a8cd5aacdfea2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/74a60f4cfd134915a9b76f41bd9a96cc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/50c4efb8ff7d44e3a280d7c21ebcd7b0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/a9dd60fc761b4988b9e679ad539b470c] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=47.7 K 2024-11-11T22:27:08,563 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:08,563 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3ef3d986844d4a199df0a55d5b5fae4c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/fef0ad7e17c04aaeabb3d19654283afc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/d708e8110b884fd9826a2792f1bbf5b4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8f364889cf3f4a10816c32ddc4464b79] 2024-11-11T22:27:08,563 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e5ccc5798134eef988a8cd5aacdfea2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731364023315 2024-11-11T22:27:08,563 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ef3d986844d4a199df0a55d5b5fae4c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731364023315 2024-11-11T22:27:08,563 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 74a60f4cfd134915a9b76f41bd9a96cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731364023530 2024-11-11T22:27:08,564 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting fef0ad7e17c04aaeabb3d19654283afc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731364023530 2024-11-11T22:27:08,564 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 50c4efb8ff7d44e3a280d7c21ebcd7b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731364024777 2024-11-11T22:27:08,564 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a9dd60fc761b4988b9e679ad539b470c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731364027119 2024-11-11T22:27:08,564 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting d708e8110b884fd9826a2792f1bbf5b4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731364024777 2024-11-11T22:27:08,564 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f364889cf3f4a10816c32ddc4464b79, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731364027119 2024-11-11T22:27:08,575 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:08,575 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/d483374391ed49b6ae78daacbdde29da is 50, key is test_row_0/B:col10/1731364027119/Put/seqid=0 2024-11-11T22:27:08,576 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:08,579 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111d944651324fe4261a7b29e40b92b5a58_1077b11658f2ec3dcbfd684d096e1030 store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:08,582 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111d944651324fe4261a7b29e40b92b5a58_1077b11658f2ec3dcbfd684d096e1030, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:08,582 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d944651324fe4261a7b29e40b92b5a58_1077b11658f2ec3dcbfd684d096e1030 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:08,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742210_1386 (size=12595) 2024-11-11T22:27:08,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742211_1387 (size=4469) 2024-11-11T22:27:08,604 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#A#compaction#328 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:08,605 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/8acbc826a9d24a36b985ca7113dc2e37 is 175, key is test_row_0/A:col10/1731364027119/Put/seqid=0 2024-11-11T22:27:08,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742212_1388 (size=31549) 2024-11-11T22:27:08,633 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/8acbc826a9d24a36b985ca7113dc2e37 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8acbc826a9d24a36b985ca7113dc2e37 2024-11-11T22:27:08,640 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/A of 1077b11658f2ec3dcbfd684d096e1030 into 8acbc826a9d24a36b985ca7113dc2e37(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:08,640 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:08,640 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/A, priority=12, startTime=1731364028561; duration=0sec 2024-11-11T22:27:08,640 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:08,640 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:A 2024-11-11T22:27:08,640 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:08,642 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:08,642 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/C is initiating minor compaction (all files) 2024-11-11T22:27:08,642 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/C in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:08,642 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/4cfe29a42a304507a3810ab956438119, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/44f7e7d5f4cb4ee9a9f9b87bf8e69cbb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/aaaf817bdd704e01b47d3399d028de9c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/5e12258543f24da391a1d423bd7090a7] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=47.7 K 2024-11-11T22:27:08,643 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cfe29a42a304507a3810ab956438119, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731364023315 2024-11-11T22:27:08,643 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44f7e7d5f4cb4ee9a9f9b87bf8e69cbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731364023530 2024-11-11T22:27:08,644 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaaf817bdd704e01b47d3399d028de9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731364024777 2024-11-11T22:27:08,644 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e12258543f24da391a1d423bd7090a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731364027119 2024-11-11T22:27:08,655 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#C#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:08,656 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/cf7779179e684394acb33e1f1b4a1a9a is 50, key is test_row_0/C:col10/1731364027119/Put/seqid=0 2024-11-11T22:27:08,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742213_1389 (size=12595) 2024-11-11T22:27:08,668 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/cf7779179e684394acb33e1f1b4a1a9a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/cf7779179e684394acb33e1f1b4a1a9a 2024-11-11T22:27:08,678 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/C of 1077b11658f2ec3dcbfd684d096e1030 into cf7779179e684394acb33e1f1b4a1a9a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:08,678 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:08,678 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/C, priority=12, startTime=1731364028561; duration=0sec 2024-11-11T22:27:08,678 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:08,678 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:C 2024-11-11T22:27:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:08,780 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:08,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:27:08,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111eb58517a6deb4d069727fc81c25311a6_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364028777/Put/seqid=0 2024-11-11T22:27:08,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742214_1390 (size=14794) 2024-11-11T22:27:08,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364088921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:08,994 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/d483374391ed49b6ae78daacbdde29da as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d483374391ed49b6ae78daacbdde29da 2024-11-11T22:27:08,999 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/B of 1077b11658f2ec3dcbfd684d096e1030 into d483374391ed49b6ae78daacbdde29da(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:08,999 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:08,999 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/B, priority=12, startTime=1731364028561; duration=0sec 2024-11-11T22:27:08,999 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:08,999 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:B 2024-11-11T22:27:09,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364089033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:09,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:09,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:09,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:27:09,154 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:09,154 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-11T22:27:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:27:09,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:09,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:09,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:09,190 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:09,192 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111eb58517a6deb4d069727fc81c25311a6_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111eb58517a6deb4d069727fc81c25311a6_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:09,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/5d21736a00cc425d99cf4f77761461d9, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:09,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/5d21736a00cc425d99cf4f77761461d9 is 175, key is test_row_0/A:col10/1731364028777/Put/seqid=0 2024-11-11T22:27:09,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742215_1391 (size=39749) 2024-11-11T22:27:09,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364089243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:09,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:09,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:09,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:27:09,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364089293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:09,297 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:09,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:27:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:09,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:27:09,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:09,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:09,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:09,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:27:09,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364089553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:09,597 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/5d21736a00cc425d99cf4f77761461d9 2024-11-11T22:27:09,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/9018fe3853a34f77a0536fd9e84ddbd6 is 50, key is test_row_0/B:col10/1731364028777/Put/seqid=0 2024-11-11T22:27:09,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742216_1392 (size=12151) 2024-11-11T22:27:09,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:27:09,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:09,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:09,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:09,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:27:09,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:09,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:09,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:09,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:27:09,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:27:09,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:09,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:09,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:09,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:10,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/9018fe3853a34f77a0536fd9e84ddbd6 2024-11-11T22:27:10,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/2146ce9960a34c7c96eeee6ac58252f1 is 50, key is test_row_0/C:col10/1731364028777/Put/seqid=0 2024-11-11T22:27:10,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742217_1393 (size=12151) 2024-11-11T22:27:10,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/2146ce9960a34c7c96eeee6ac58252f1 2024-11-11T22:27:10,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/5d21736a00cc425d99cf4f77761461d9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5d21736a00cc425d99cf4f77761461d9 2024-11-11T22:27:10,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5d21736a00cc425d99cf4f77761461d9, entries=200, sequenceid=211, filesize=38.8 K 2024-11-11T22:27:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/9018fe3853a34f77a0536fd9e84ddbd6 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/9018fe3853a34f77a0536fd9e84ddbd6 2024-11-11T22:27:10,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/9018fe3853a34f77a0536fd9e84ddbd6, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T22:27:10,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364090064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:10,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:10,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:10,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:10,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:27:10,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:10,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:10,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:10,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:10,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:10,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/2146ce9960a34c7c96eeee6ac58252f1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2146ce9960a34c7c96eeee6ac58252f1 2024-11-11T22:27:10,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2146ce9960a34c7c96eeee6ac58252f1, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T22:27:10,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1077b11658f2ec3dcbfd684d096e1030 in 1302ms, sequenceid=211, compaction requested=false 2024-11-11T22:27:10,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:10,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:27:10,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:10,227 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:27:10,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117a62698f2d5947bda75890853f6c700a_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364028920/Put/seqid=0 2024-11-11T22:27:10,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742218_1394 (size=12304) 2024-11-11T22:27:10,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:10,245 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117a62698f2d5947bda75890853f6c700a_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117a62698f2d5947bda75890853f6c700a_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/94ae519486984913b37cde0ab7bd586b, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/94ae519486984913b37cde0ab7bd586b is 175, key is test_row_0/A:col10/1731364028920/Put/seqid=0 2024-11-11T22:27:10,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742219_1395 (size=31105) 2024-11-11T22:27:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:27:10,660 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/94ae519486984913b37cde0ab7bd586b 2024-11-11T22:27:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8b3710ba9fcd42c3ab4bc96d112a25ab is 50, key is test_row_0/B:col10/1731364028920/Put/seqid=0 2024-11-11T22:27:10,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742220_1396 (size=12151) 2024-11-11T22:27:11,073 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8b3710ba9fcd42c3ab4bc96d112a25ab 2024-11-11T22:27:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/37edb9c31774408096cb5541e6e6bacc is 50, key is test_row_0/C:col10/1731364028920/Put/seqid=0 2024-11-11T22:27:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:11,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:11,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742221_1397 (size=12151) 2024-11-11T22:27:11,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364091128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:11,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364091243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:27:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364091323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:11,326 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 1077b11658f2ec3dcbfd684d096e1030 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:11,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364091453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:11,516 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/37edb9c31774408096cb5541e6e6bacc 2024-11-11T22:27:11,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/94ae519486984913b37cde0ab7bd586b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/94ae519486984913b37cde0ab7bd586b 2024-11-11T22:27:11,525 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/94ae519486984913b37cde0ab7bd586b, entries=150, sequenceid=237, filesize=30.4 K 2024-11-11T22:27:11,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8b3710ba9fcd42c3ab4bc96d112a25ab as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8b3710ba9fcd42c3ab4bc96d112a25ab 2024-11-11T22:27:11,530 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8b3710ba9fcd42c3ab4bc96d112a25ab, entries=150, sequenceid=237, filesize=11.9 K 2024-11-11T22:27:11,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/37edb9c31774408096cb5541e6e6bacc as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/37edb9c31774408096cb5541e6e6bacc 2024-11-11T22:27:11,534 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/37edb9c31774408096cb5541e6e6bacc, entries=150, sequenceid=237, filesize=11.9 K 2024-11-11T22:27:11,535 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1077b11658f2ec3dcbfd684d096e1030 in 1308ms, sequenceid=237, compaction requested=true 2024-11-11T22:27:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T22:27:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-11T22:27:11,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T22:27:11,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3810 sec 2024-11-11T22:27:11,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3840 sec 2024-11-11T22:27:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:11,768 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:11,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:27:11,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fca5e5929787423b8d6432ca33086aa1_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364031766/Put/seqid=0 2024-11-11T22:27:11,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742222_1398 (size=14794) 2024-11-11T22:27:11,790 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:11,793 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fca5e5929787423b8d6432ca33086aa1_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fca5e5929787423b8d6432ca33086aa1_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:11,793 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/9359f51051b24e2695d64fbe35c946ea, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:11,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/9359f51051b24e2695d64fbe35c946ea is 175, key is test_row_0/A:col10/1731364031766/Put/seqid=0 2024-11-11T22:27:11,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742223_1399 (size=39749) 2024-11-11T22:27:11,797 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/9359f51051b24e2695d64fbe35c946ea 2024-11-11T22:27:11,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/3595f97752e44a2c83dd39a4607406b4 is 50, key is test_row_0/B:col10/1731364031766/Put/seqid=0 2024-11-11T22:27:11,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742224_1400 (size=12151) 2024-11-11T22:27:11,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364091912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364092023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:12,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/3595f97752e44a2c83dd39a4607406b4 2024-11-11T22:27:12,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/fca503f4ffe341ef8a29943350cde0bd is 50, key is test_row_0/C:col10/1731364031766/Put/seqid=0 2024-11-11T22:27:12,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742225_1401 (size=12151) 2024-11-11T22:27:12,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364092233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:12,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:12,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:12,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364092554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:12,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/fca503f4ffe341ef8a29943350cde0bd 2024-11-11T22:27:12,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/9359f51051b24e2695d64fbe35c946ea as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/9359f51051b24e2695d64fbe35c946ea 2024-11-11T22:27:12,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/9359f51051b24e2695d64fbe35c946ea, entries=200, sequenceid=249, filesize=38.8 K 2024-11-11T22:27:12,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/3595f97752e44a2c83dd39a4607406b4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/3595f97752e44a2c83dd39a4607406b4 2024-11-11T22:27:12,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/3595f97752e44a2c83dd39a4607406b4, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T22:27:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/fca503f4ffe341ef8a29943350cde0bd as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/fca503f4ffe341ef8a29943350cde0bd 2024-11-11T22:27:12,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/fca503f4ffe341ef8a29943350cde0bd, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T22:27:12,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1077b11658f2ec3dcbfd684d096e1030 in 869ms, sequenceid=249, compaction requested=true 2024-11-11T22:27:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:12,637 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:12,637 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:12,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:12,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:12,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:12,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:12,638 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:12,638 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:12,638 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/B is initiating minor compaction (all files) 2024-11-11T22:27:12,638 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/A is initiating minor compaction (all files) 2024-11-11T22:27:12,638 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/A in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:12,638 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/B in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:12,639 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d483374391ed49b6ae78daacbdde29da, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/9018fe3853a34f77a0536fd9e84ddbd6, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8b3710ba9fcd42c3ab4bc96d112a25ab, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/3595f97752e44a2c83dd39a4607406b4] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=47.9 K 2024-11-11T22:27:12,639 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8acbc826a9d24a36b985ca7113dc2e37, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5d21736a00cc425d99cf4f77761461d9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/94ae519486984913b37cde0ab7bd586b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/9359f51051b24e2695d64fbe35c946ea] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=138.8 K 2024-11-11T22:27:12,639 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:12,639 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8acbc826a9d24a36b985ca7113dc2e37, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5d21736a00cc425d99cf4f77761461d9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/94ae519486984913b37cde0ab7bd586b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/9359f51051b24e2695d64fbe35c946ea] 2024-11-11T22:27:12,639 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8acbc826a9d24a36b985ca7113dc2e37, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731364027119 2024-11-11T22:27:12,639 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting d483374391ed49b6ae78daacbdde29da, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731364027119 2024-11-11T22:27:12,639 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d21736a00cc425d99cf4f77761461d9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731364028103 2024-11-11T22:27:12,639 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 9018fe3853a34f77a0536fd9e84ddbd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731364028111 2024-11-11T22:27:12,640 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b3710ba9fcd42c3ab4bc96d112a25ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731364028874 2024-11-11T22:27:12,640 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94ae519486984913b37cde0ab7bd586b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731364028874 2024-11-11T22:27:12,640 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 3595f97752e44a2c83dd39a4607406b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731364031093 2024-11-11T22:27:12,640 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9359f51051b24e2695d64fbe35c946ea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731364031093 2024-11-11T22:27:12,650 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:12,653 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#B#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:12,653 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/d43e4f6d6967464cba5f8f0227fdcaef is 50, key is test_row_0/B:col10/1731364031766/Put/seqid=0 2024-11-11T22:27:12,661 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111ef35ecb450e44ef298ae4f714ec85c4e_1077b11658f2ec3dcbfd684d096e1030 store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:12,663 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111ef35ecb450e44ef298ae4f714ec85c4e_1077b11658f2ec3dcbfd684d096e1030, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:12,663 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ef35ecb450e44ef298ae4f714ec85c4e_1077b11658f2ec3dcbfd684d096e1030 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:12,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742226_1402 (size=12731) 2024-11-11T22:27:12,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742227_1403 (size=4469) 2024-11-11T22:27:12,678 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/d43e4f6d6967464cba5f8f0227fdcaef as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d43e4f6d6967464cba5f8f0227fdcaef 2024-11-11T22:27:12,681 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#A#compaction#339 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:12,682 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/1c03c6d7eaf84c35b91ff92300bdacb4 is 175, key is test_row_0/A:col10/1731364031766/Put/seqid=0 2024-11-11T22:27:12,690 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/B of 1077b11658f2ec3dcbfd684d096e1030 into d43e4f6d6967464cba5f8f0227fdcaef(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:12,690 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:12,690 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/B, priority=12, startTime=1731364032637; duration=0sec 2024-11-11T22:27:12,690 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:12,690 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:B 2024-11-11T22:27:12,690 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:12,692 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:12,692 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/C is initiating minor compaction (all files) 2024-11-11T22:27:12,692 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/C in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:12,692 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/cf7779179e684394acb33e1f1b4a1a9a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2146ce9960a34c7c96eeee6ac58252f1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/37edb9c31774408096cb5541e6e6bacc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/fca503f4ffe341ef8a29943350cde0bd] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=47.9 K 2024-11-11T22:27:12,692 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting cf7779179e684394acb33e1f1b4a1a9a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731364027119 2024-11-11T22:27:12,693 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 2146ce9960a34c7c96eeee6ac58252f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731364028111 2024-11-11T22:27:12,693 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 37edb9c31774408096cb5541e6e6bacc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731364028874 2024-11-11T22:27:12,693 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting fca503f4ffe341ef8a29943350cde0bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731364031093 2024-11-11T22:27:12,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742228_1404 (size=31685) 2024-11-11T22:27:12,706 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:12,706 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/affc6482274a48e18e0ec8f8562a6a06 is 50, key is test_row_0/C:col10/1731364031766/Put/seqid=0 2024-11-11T22:27:12,707 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/1c03c6d7eaf84c35b91ff92300bdacb4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1c03c6d7eaf84c35b91ff92300bdacb4 2024-11-11T22:27:12,713 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/A of 1077b11658f2ec3dcbfd684d096e1030 into 1c03c6d7eaf84c35b91ff92300bdacb4(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:12,713 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:12,713 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/A, priority=12, startTime=1731364032637; duration=0sec 2024-11-11T22:27:12,713 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:12,713 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:A 2024-11-11T22:27:12,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742229_1405 (size=12731) 2024-11-11T22:27:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:13,078 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:13,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:27:13,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115f7eebbaea894002b5d078a7791db4b7_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364033077/Put/seqid=0 2024-11-11T22:27:13,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742230_1406 (size=14994) 2024-11-11T22:27:13,118 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/affc6482274a48e18e0ec8f8562a6a06 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/affc6482274a48e18e0ec8f8562a6a06 2024-11-11T22:27:13,122 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/C of 1077b11658f2ec3dcbfd684d096e1030 into affc6482274a48e18e0ec8f8562a6a06(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:13,122 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:13,122 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/C, priority=12, startTime=1731364032638; duration=0sec 2024-11-11T22:27:13,122 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:13,122 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:C 2024-11-11T22:27:13,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:13,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364093133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:13,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:13,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364093243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:27:13,314 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:13,314 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-11T22:27:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:27:13,317 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:13,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:13,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:27:13,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364093453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:13,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:27:13,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:13,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:13,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:13,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,492 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:13,495 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115f7eebbaea894002b5d078a7791db4b7_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f7eebbaea894002b5d078a7791db4b7_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:13,496 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/b91d01bc904f4261b470a0b493faf1df, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:13,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/b91d01bc904f4261b470a0b493faf1df is 175, key is test_row_0/A:col10/1731364033077/Put/seqid=0 2024-11-11T22:27:13,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742231_1407 (size=39949) 2024-11-11T22:27:13,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:27:13,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:13,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:13,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:13,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:27:13,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364093763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:13,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:27:13,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:13,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:13,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:13,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,899 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/b91d01bc904f4261b470a0b493faf1df 2024-11-11T22:27:13,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/4375de428e6b43e9aa3d276e4ce0017c is 50, key is test_row_0/B:col10/1731364033077/Put/seqid=0 2024-11-11T22:27:13,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742232_1408 (size=12301) 2024-11-11T22:27:13,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:27:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:13,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:27:14,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:27:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:27:14,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:14,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364094283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:14,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/4375de428e6b43e9aa3d276e4ce0017c 2024-11-11T22:27:14,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/2b2e0c7042024357b2af42dc4745429b is 50, key is test_row_0/C:col10/1731364033077/Put/seqid=0 2024-11-11T22:27:14,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742233_1409 (size=12301) 2024-11-11T22:27:14,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:27:14,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:14,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:27:14,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:27:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:27:14,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:14,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:14,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/2b2e0c7042024357b2af42dc4745429b 2024-11-11T22:27:14,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/b91d01bc904f4261b470a0b493faf1df as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/b91d01bc904f4261b470a0b493faf1df 2024-11-11T22:27:14,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/b91d01bc904f4261b470a0b493faf1df, entries=200, sequenceid=276, filesize=39.0 K 2024-11-11T22:27:14,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/4375de428e6b43e9aa3d276e4ce0017c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4375de428e6b43e9aa3d276e4ce0017c 2024-11-11T22:27:14,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4375de428e6b43e9aa3d276e4ce0017c, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T22:27:14,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/2b2e0c7042024357b2af42dc4745429b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2b2e0c7042024357b2af42dc4745429b 2024-11-11T22:27:14,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2b2e0c7042024357b2af42dc4745429b, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T22:27:14,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1077b11658f2ec3dcbfd684d096e1030 in 1663ms, sequenceid=276, compaction requested=false 2024-11-11T22:27:14,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:14,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:27:14,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:14,845 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:27:14,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110dc565935e0f4094b297574cfce6dd63_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364033124/Put/seqid=0 2024-11-11T22:27:14,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742234_1410 (size=12454) 2024-11-11T22:27:15,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:15,268 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110dc565935e0f4094b297574cfce6dd63_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110dc565935e0f4094b297574cfce6dd63_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:15,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/bbd25cb830a2483592db8f7531b2e361, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:15,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/bbd25cb830a2483592db8f7531b2e361 is 175, key is test_row_0/A:col10/1731364033124/Put/seqid=0 2024-11-11T22:27:15,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742235_1411 (size=31255) 2024-11-11T22:27:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:15,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. as already flushing 2024-11-11T22:27:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364095402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364095404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:27:15,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364095513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:15,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364095523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:15,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:15,678 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/bbd25cb830a2483592db8f7531b2e361 2024-11-11T22:27:15,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8d548e8e7c9f4c8e8fa996febeffeee1 is 50, key is test_row_0/B:col10/1731364033124/Put/seqid=0 2024-11-11T22:27:15,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742236_1412 (size=12301) 2024-11-11T22:27:15,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364095722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:15,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364095743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:16,032 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:27:16,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36052 deadline: 1731364096033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:16,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364096063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:16,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8d548e8e7c9f4c8e8fa996febeffeee1 2024-11-11T22:27:16,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/2c3524f1be5e4f42a6e873c635bae20c is 50, key is test_row_0/C:col10/1731364033124/Put/seqid=0 2024-11-11T22:27:16,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742237_1413 (size=12301) 2024-11-11T22:27:16,100 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/2c3524f1be5e4f42a6e873c635bae20c 2024-11-11T22:27:16,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/bbd25cb830a2483592db8f7531b2e361 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/bbd25cb830a2483592db8f7531b2e361 2024-11-11T22:27:16,112 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/bbd25cb830a2483592db8f7531b2e361, entries=150, sequenceid=288, filesize=30.5 K 2024-11-11T22:27:16,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8d548e8e7c9f4c8e8fa996febeffeee1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8d548e8e7c9f4c8e8fa996febeffeee1 2024-11-11T22:27:16,116 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8d548e8e7c9f4c8e8fa996febeffeee1, entries=150, sequenceid=288, filesize=12.0 K 2024-11-11T22:27:16,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/2c3524f1be5e4f42a6e873c635bae20c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2c3524f1be5e4f42a6e873c635bae20c 2024-11-11T22:27:16,122 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2c3524f1be5e4f42a6e873c635bae20c, entries=150, sequenceid=288, filesize=12.0 K 2024-11-11T22:27:16,123 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1077b11658f2ec3dcbfd684d096e1030 in 1278ms, sequenceid=288, compaction requested=true 2024-11-11T22:27:16,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:16,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:16,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-11T22:27:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-11T22:27:16,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T22:27:16,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8070 sec 2024-11-11T22:27:16,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.8140 sec 2024-11-11T22:27:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:16,569 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:16,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T22:27:16,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111291d6c05d6194497937641628d583022_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364036547/Put/seqid=0 2024-11-11T22:27:16,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742238_1414 (size=14994) 2024-11-11T22:27:16,584 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:16,587 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111291d6c05d6194497937641628d583022_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111291d6c05d6194497937641628d583022_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:16,587 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/da5613a978594fa59b2d4bbf31bd6fbe, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:16,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/da5613a978594fa59b2d4bbf31bd6fbe is 175, key is test_row_0/A:col10/1731364036547/Put/seqid=0 2024-11-11T22:27:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742239_1415 (size=39949) 2024-11-11T22:27:16,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/da5613a978594fa59b2d4bbf31bd6fbe 2024-11-11T22:27:16,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/0aea2930f1484db48db41d03f38cb856 is 50, key is test_row_0/B:col10/1731364036547/Put/seqid=0 2024-11-11T22:27:16,606 INFO [Thread-1520 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1520. 2024-11-11T22:27:16,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:16,606 DEBUG [Thread-1520 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:16,606 DEBUG [Thread-1520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1731364096598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:16,606 DEBUG [Thread-1520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:16,606 INFO [Thread-1531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1531. 2024-11-11T22:27:16,606 DEBUG [Thread-1531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:16,606 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,607 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:16,607 INFO [Thread-1539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1539. 2024-11-11T22:27:16,607 DEBUG [Thread-1539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:16,607 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,607 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:16,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742240_1416 (size=12301) 2024-11-11T22:27:16,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/0aea2930f1484db48db41d03f38cb856 2024-11-11T22:27:16,610 INFO [Thread-1537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1537. 2024-11-11T22:27:16,610 DEBUG [Thread-1537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:16,610 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,611 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:16,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/1b9db5049b59412eb663e5710efc8100 is 50, key is test_row_0/C:col10/1731364036547/Put/seqid=0 2024-11-11T22:27:16,617 INFO [Thread-1533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1533. 2024-11-11T22:27:16,618 DEBUG [Thread-1533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:16,618 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,618 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:16,619 INFO [Thread-1535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1535. 2024-11-11T22:27:16,619 DEBUG [Thread-1535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:16,619 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,619 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:16,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742241_1417 (size=12301) 2024-11-11T22:27:16,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/1b9db5049b59412eb663e5710efc8100 2024-11-11T22:27:16,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/da5613a978594fa59b2d4bbf31bd6fbe as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/da5613a978594fa59b2d4bbf31bd6fbe 2024-11-11T22:27:16,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/da5613a978594fa59b2d4bbf31bd6fbe, entries=200, sequenceid=314, filesize=39.0 K 2024-11-11T22:27:16,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/0aea2930f1484db48db41d03f38cb856 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/0aea2930f1484db48db41d03f38cb856 2024-11-11T22:27:16,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/0aea2930f1484db48db41d03f38cb856, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T22:27:16,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/1b9db5049b59412eb663e5710efc8100 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1b9db5049b59412eb663e5710efc8100 2024-11-11T22:27:16,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1b9db5049b59412eb663e5710efc8100, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T22:27:16,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1077b11658f2ec3dcbfd684d096e1030 in 71ms, sequenceid=314, compaction requested=true 2024-11-11T22:27:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:16,641 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1077b11658f2ec3dcbfd684d096e1030:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:16,641 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:16,642 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142838 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:16,642 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:16,642 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/B is initiating minor compaction (all files) 2024-11-11T22:27:16,642 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/A is initiating minor compaction (all files) 2024-11-11T22:27:16,642 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/A in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:16,642 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/B in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:16,642 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1c03c6d7eaf84c35b91ff92300bdacb4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/b91d01bc904f4261b470a0b493faf1df, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/bbd25cb830a2483592db8f7531b2e361, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/da5613a978594fa59b2d4bbf31bd6fbe] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=139.5 K 2024-11-11T22:27:16,642 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d43e4f6d6967464cba5f8f0227fdcaef, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4375de428e6b43e9aa3d276e4ce0017c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8d548e8e7c9f4c8e8fa996febeffeee1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/0aea2930f1484db48db41d03f38cb856] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=48.5 K 2024-11-11T22:27:16,642 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:16,642 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1c03c6d7eaf84c35b91ff92300bdacb4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/b91d01bc904f4261b470a0b493faf1df, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/bbd25cb830a2483592db8f7531b2e361, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/da5613a978594fa59b2d4bbf31bd6fbe] 2024-11-11T22:27:16,642 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting d43e4f6d6967464cba5f8f0227fdcaef, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731364031093 2024-11-11T22:27:16,642 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c03c6d7eaf84c35b91ff92300bdacb4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731364031093 2024-11-11T22:27:16,642 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4375de428e6b43e9aa3d276e4ce0017c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731364031888 2024-11-11T22:27:16,642 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b91d01bc904f4261b470a0b493faf1df, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731364031867 2024-11-11T22:27:16,643 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d548e8e7c9f4c8e8fa996febeffeee1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731364033114 2024-11-11T22:27:16,643 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbd25cb830a2483592db8f7531b2e361, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731364033114 2024-11-11T22:27:16,643 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aea2930f1484db48db41d03f38cb856, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731364035394 2024-11-11T22:27:16,643 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting da5613a978594fa59b2d4bbf31bd6fbe, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731364035394 2024-11-11T22:27:16,650 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#B#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:16,651 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8e668d656efb44c0a24df49950f51c22 is 50, key is test_row_0/B:col10/1731364036547/Put/seqid=0 2024-11-11T22:27:16,651 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:16,657 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411115cd35271e2ba4a8bb1da2c392ccaf3e4_1077b11658f2ec3dcbfd684d096e1030 store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:16,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742242_1418 (size=13017) 2024-11-11T22:27:16,672 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411115cd35271e2ba4a8bb1da2c392ccaf3e4_1077b11658f2ec3dcbfd684d096e1030, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:16,672 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115cd35271e2ba4a8bb1da2c392ccaf3e4_1077b11658f2ec3dcbfd684d096e1030 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:16,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742243_1419 (size=4469) 2024-11-11T22:27:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:16,718 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:16,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:27:16,719 INFO [Thread-1526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1526. 2024-11-11T22:27:16,719 DEBUG [Thread-1526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:16,719 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,719 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:16,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:16,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cc17efb3c7ee4b6da413ad4a848e9be1_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364036713/Put/seqid=0 2024-11-11T22:27:16,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742244_1420 (size=12454) 2024-11-11T22:27:17,064 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/8e668d656efb44c0a24df49950f51c22 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8e668d656efb44c0a24df49950f51c22 2024-11-11T22:27:17,068 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/B of 1077b11658f2ec3dcbfd684d096e1030 into 8e668d656efb44c0a24df49950f51c22(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:17,068 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:17,068 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/B, priority=12, startTime=1731364036641; duration=0sec 2024-11-11T22:27:17,069 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:17,069 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:B 2024-11-11T22:27:17,069 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:17,070 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:17,070 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 1077b11658f2ec3dcbfd684d096e1030/C is initiating minor compaction (all files) 2024-11-11T22:27:17,070 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1077b11658f2ec3dcbfd684d096e1030/C in TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:17,070 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/affc6482274a48e18e0ec8f8562a6a06, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2b2e0c7042024357b2af42dc4745429b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2c3524f1be5e4f42a6e873c635bae20c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1b9db5049b59412eb663e5710efc8100] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp, totalSize=48.5 K 2024-11-11T22:27:17,070 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting affc6482274a48e18e0ec8f8562a6a06, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731364031093 2024-11-11T22:27:17,071 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2e0c7042024357b2af42dc4745429b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731364031888 2024-11-11T22:27:17,071 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c3524f1be5e4f42a6e873c635bae20c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731364033114 2024-11-11T22:27:17,071 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9db5049b59412eb663e5710efc8100, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731364035394 2024-11-11T22:27:17,077 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#A#compaction#352 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:17,077 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/c7045b29a2c14c28b49bd2d26d7a67ff is 175, key is test_row_0/A:col10/1731364036547/Put/seqid=0 2024-11-11T22:27:17,079 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1077b11658f2ec3dcbfd684d096e1030#C#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:17,080 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/3d0c2cfdc6904e69b50c8d5bc3f5fd45 is 50, key is test_row_0/C:col10/1731364036547/Put/seqid=0 2024-11-11T22:27:17,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742245_1421 (size=31971) 2024-11-11T22:27:17,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742246_1422 (size=13017) 2024-11-11T22:27:17,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:17,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cc17efb3c7ee4b6da413ad4a848e9be1_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cc17efb3c7ee4b6da413ad4a848e9be1_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:17,135 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/12ee55b9553c47ceb36c82b78d8aebae, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:17,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/12ee55b9553c47ceb36c82b78d8aebae is 175, key is test_row_0/A:col10/1731364036713/Put/seqid=0 2024-11-11T22:27:17,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742247_1423 (size=31255) 2024-11-11T22:27:17,485 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/c7045b29a2c14c28b49bd2d26d7a67ff as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/c7045b29a2c14c28b49bd2d26d7a67ff 2024-11-11T22:27:17,485 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/3d0c2cfdc6904e69b50c8d5bc3f5fd45 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/3d0c2cfdc6904e69b50c8d5bc3f5fd45 2024-11-11T22:27:17,489 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/A of 1077b11658f2ec3dcbfd684d096e1030 into c7045b29a2c14c28b49bd2d26d7a67ff(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:17,489 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1077b11658f2ec3dcbfd684d096e1030/C of 1077b11658f2ec3dcbfd684d096e1030 into 3d0c2cfdc6904e69b50c8d5bc3f5fd45(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:17,489 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:17,489 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:17,489 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/A, priority=12, startTime=1731364036641; duration=0sec 2024-11-11T22:27:17,489 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., storeName=1077b11658f2ec3dcbfd684d096e1030/C, priority=12, startTime=1731364036641; duration=0sec 2024-11-11T22:27:17,489 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:17,489 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:C 2024-11-11T22:27:17,489 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:17,489 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1077b11658f2ec3dcbfd684d096e1030:A 2024-11-11T22:27:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:27:17,494 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:17,538 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/12ee55b9553c47ceb36c82b78d8aebae 2024-11-11T22:27:17,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/f211498b770645db842ab6c95a39907f is 50, key is test_row_0/B:col10/1731364036713/Put/seqid=0 2024-11-11T22:27:17,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742248_1424 (size=12301) 2024-11-11T22:27:17,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/f211498b770645db842ab6c95a39907f 2024-11-11T22:27:17,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/637d708fd69544e69eeca003607d44ff is 50, key is test_row_0/C:col10/1731364036713/Put/seqid=0 2024-11-11T22:27:17,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742249_1425 (size=12301) 2024-11-11T22:27:18,175 INFO [Thread-1524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1524. 2024-11-11T22:27:18,175 DEBUG [Thread-1524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:18,175 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:18,175 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:18,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:18,194 INFO [Thread-1528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1528. 2024-11-11T22:27:18,194 DEBUG [Thread-1528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:18,194 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:18,194 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:18,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:18,204 INFO [Thread-1522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1522. 2024-11-11T22:27:18,204 DEBUG [Thread-1522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:18,204 DEBUG [Thread-1522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:18,204 DEBUG [Thread-1522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:18,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1386 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4138 rows 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1383 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4133 rows 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1377 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4119 rows 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1388 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4157 rows 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1396 2024-11-11T22:27:18,205 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4164 rows 2024-11-11T22:27:18,205 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:27:18,205 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:27:18,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:18,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:18,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:18,206 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:27:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:27:18,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364038208"}]},"ts":"1731364038208"} 2024-11-11T22:27:18,209 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:27:18,210 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:27:18,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:27:18,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, UNASSIGN}] 2024-11-11T22:27:18,212 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, UNASSIGN 2024-11-11T22:27:18,213 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=1077b11658f2ec3dcbfd684d096e1030, regionState=CLOSING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:18,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, UNASSIGN because future has completed 2024-11-11T22:27:18,214 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:27:18,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:27:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:27:18,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/637d708fd69544e69eeca003607d44ff 2024-11-11T22:27:18,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/12ee55b9553c47ceb36c82b78d8aebae as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/12ee55b9553c47ceb36c82b78d8aebae 2024-11-11T22:27:18,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/12ee55b9553c47ceb36c82b78d8aebae, entries=150, sequenceid=325, filesize=30.5 K 2024-11-11T22:27:18,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/f211498b770645db842ab6c95a39907f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/f211498b770645db842ab6c95a39907f 2024-11-11T22:27:18,366 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:18,366 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:27:18,366 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing 1077b11658f2ec3dcbfd684d096e1030, disabling compactions & flushes 2024-11-11T22:27:18,366 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:18,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/f211498b770645db842ab6c95a39907f, entries=150, sequenceid=325, filesize=12.0 K 2024-11-11T22:27:18,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/637d708fd69544e69eeca003607d44ff as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/637d708fd69544e69eeca003607d44ff 2024-11-11T22:27:18,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/637d708fd69544e69eeca003607d44ff, entries=150, sequenceid=325, filesize=12.0 K 2024-11-11T22:27:18,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=20.13 KB/20610 for 1077b11658f2ec3dcbfd684d096e1030 in 1655ms, sequenceid=325, compaction requested=false 2024-11-11T22:27:18,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1077b11658f2ec3dcbfd684d096e1030: 2024-11-11T22:27:18,373 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:18,373 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:18,373 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. after waiting 0 ms 2024-11-11T22:27:18,374 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:18,374 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing 1077b11658f2ec3dcbfd684d096e1030 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-11T22:27:18,378 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dd6b4cd39571448fae33db851ba6e400_1077b11658f2ec3dcbfd684d096e1030 is 50, key is test_row_0/A:col10/1731364038193/Put/seqid=0 2024-11-11T22:27:18,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742250_1426 (size=9914) 2024-11-11T22:27:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:27:18,781 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:18,785 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dd6b4cd39571448fae33db851ba6e400_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dd6b4cd39571448fae33db851ba6e400_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:18,785 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/698a1f8c872b4fd990fc7a46260c13fe, store: [table=TestAcidGuarantees family=A region=1077b11658f2ec3dcbfd684d096e1030] 2024-11-11T22:27:18,786 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/698a1f8c872b4fd990fc7a46260c13fe is 175, key is test_row_0/A:col10/1731364038193/Put/seqid=0 2024-11-11T22:27:18,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742251_1427 (size=22561) 2024-11-11T22:27:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:27:19,190 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/698a1f8c872b4fd990fc7a46260c13fe 2024-11-11T22:27:19,195 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/a6fb66de146443d6abc97c14c4b0cd99 is 50, key is test_row_0/B:col10/1731364038193/Put/seqid=0 2024-11-11T22:27:19,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742252_1428 (size=9857) 2024-11-11T22:27:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:27:19,599 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/a6fb66de146443d6abc97c14c4b0cd99 2024-11-11T22:27:19,604 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/fca37aef6f6046d5a69cf12d529f7786 is 50, key is test_row_0/C:col10/1731364038193/Put/seqid=0 2024-11-11T22:27:19,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742253_1429 (size=9857) 2024-11-11T22:27:20,008 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/fca37aef6f6046d5a69cf12d529f7786 2024-11-11T22:27:20,011 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/A/698a1f8c872b4fd990fc7a46260c13fe as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/698a1f8c872b4fd990fc7a46260c13fe 2024-11-11T22:27:20,013 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/698a1f8c872b4fd990fc7a46260c13fe, entries=100, sequenceid=334, filesize=22.0 K 2024-11-11T22:27:20,014 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/B/a6fb66de146443d6abc97c14c4b0cd99 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/a6fb66de146443d6abc97c14c4b0cd99 2024-11-11T22:27:20,017 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/a6fb66de146443d6abc97c14c4b0cd99, entries=100, sequenceid=334, filesize=9.6 K 2024-11-11T22:27:20,017 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/.tmp/C/fca37aef6f6046d5a69cf12d529f7786 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/fca37aef6f6046d5a69cf12d529f7786 2024-11-11T22:27:20,020 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/fca37aef6f6046d5a69cf12d529f7786, entries=100, sequenceid=334, filesize=9.6 K 2024-11-11T22:27:20,021 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 1077b11658f2ec3dcbfd684d096e1030 in 1646ms, sequenceid=334, compaction requested=true 2024-11-11T22:27:20,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/ece7085753fc4ecf8d5f88e2d3a2e0c5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1b8c3eb0a6e04c9196982bb23028a8b1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5179bef741304c7e9c8904bbcb251828, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3014c823c1a049e38f47dd492820a283, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/39524dfbd9ce465f83bd9e391950f296, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/2807f3fe37ea43fa820ebc68cb7eb3ef, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1a9b42b97b7f4b1690da12698a3995de, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/cd9810cba29342a3a12a41f428ca0d65, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/31e3b9849d32461b8ffd7c2833bf4609, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3ef3d986844d4a199df0a55d5b5fae4c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/fef0ad7e17c04aaeabb3d19654283afc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/d708e8110b884fd9826a2792f1bbf5b4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8f364889cf3f4a10816c32ddc4464b79, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8acbc826a9d24a36b985ca7113dc2e37, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5d21736a00cc425d99cf4f77761461d9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/94ae519486984913b37cde0ab7bd586b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/9359f51051b24e2695d64fbe35c946ea, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1c03c6d7eaf84c35b91ff92300bdacb4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/b91d01bc904f4261b470a0b493faf1df, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/bbd25cb830a2483592db8f7531b2e361, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/da5613a978594fa59b2d4bbf31bd6fbe] to archive 2024-11-11T22:27:20,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:27:20,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/ece7085753fc4ecf8d5f88e2d3a2e0c5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/ece7085753fc4ecf8d5f88e2d3a2e0c5 2024-11-11T22:27:20,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1b8c3eb0a6e04c9196982bb23028a8b1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1b8c3eb0a6e04c9196982bb23028a8b1 2024-11-11T22:27:20,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5179bef741304c7e9c8904bbcb251828 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5179bef741304c7e9c8904bbcb251828 2024-11-11T22:27:20,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3014c823c1a049e38f47dd492820a283 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3014c823c1a049e38f47dd492820a283 2024-11-11T22:27:20,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/39524dfbd9ce465f83bd9e391950f296 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/39524dfbd9ce465f83bd9e391950f296 2024-11-11T22:27:20,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/2807f3fe37ea43fa820ebc68cb7eb3ef to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/2807f3fe37ea43fa820ebc68cb7eb3ef 2024-11-11T22:27:20,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1a9b42b97b7f4b1690da12698a3995de to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1a9b42b97b7f4b1690da12698a3995de 2024-11-11T22:27:20,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/cd9810cba29342a3a12a41f428ca0d65 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/cd9810cba29342a3a12a41f428ca0d65 2024-11-11T22:27:20,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/31e3b9849d32461b8ffd7c2833bf4609 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/31e3b9849d32461b8ffd7c2833bf4609 2024-11-11T22:27:20,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3ef3d986844d4a199df0a55d5b5fae4c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/3ef3d986844d4a199df0a55d5b5fae4c 2024-11-11T22:27:20,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/fef0ad7e17c04aaeabb3d19654283afc to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/fef0ad7e17c04aaeabb3d19654283afc 2024-11-11T22:27:20,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/d708e8110b884fd9826a2792f1bbf5b4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/d708e8110b884fd9826a2792f1bbf5b4 2024-11-11T22:27:20,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8f364889cf3f4a10816c32ddc4464b79 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8f364889cf3f4a10816c32ddc4464b79 2024-11-11T22:27:20,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8acbc826a9d24a36b985ca7113dc2e37 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/8acbc826a9d24a36b985ca7113dc2e37 2024-11-11T22:27:20,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5d21736a00cc425d99cf4f77761461d9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/5d21736a00cc425d99cf4f77761461d9 2024-11-11T22:27:20,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/94ae519486984913b37cde0ab7bd586b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/94ae519486984913b37cde0ab7bd586b 2024-11-11T22:27:20,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/9359f51051b24e2695d64fbe35c946ea to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/9359f51051b24e2695d64fbe35c946ea 2024-11-11T22:27:20,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1c03c6d7eaf84c35b91ff92300bdacb4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/1c03c6d7eaf84c35b91ff92300bdacb4 2024-11-11T22:27:20,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/b91d01bc904f4261b470a0b493faf1df to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/b91d01bc904f4261b470a0b493faf1df 2024-11-11T22:27:20,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/bbd25cb830a2483592db8f7531b2e361 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/bbd25cb830a2483592db8f7531b2e361 2024-11-11T22:27:20,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/da5613a978594fa59b2d4bbf31bd6fbe to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/da5613a978594fa59b2d4bbf31bd6fbe 2024-11-11T22:27:20,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4aa74d0ebeee4181b04d746cc16bf9b0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8bb6207d86694a7cb801a1829d0569b5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/99a5c3d80fc04b14b94b99985e383460, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/894705c39dc3434a80bf2c3143e59e40, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/bad7e3f37f5d40b38dc972ac2f49244e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/b3689f75fb9e41f082a64a2d1adf2bfe, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/5f59e896a08c4da1b80cac09c6d41fea, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d98384b9fbd744d5a4c81de5a7b9a09c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8e5ccc5798134eef988a8cd5aacdfea2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/6b27b8ac858d47d6888aaa5a7c6c69da, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/74a60f4cfd134915a9b76f41bd9a96cc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/50c4efb8ff7d44e3a280d7c21ebcd7b0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d483374391ed49b6ae78daacbdde29da, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/a9dd60fc761b4988b9e679ad539b470c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/9018fe3853a34f77a0536fd9e84ddbd6, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8b3710ba9fcd42c3ab4bc96d112a25ab, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d43e4f6d6967464cba5f8f0227fdcaef, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/3595f97752e44a2c83dd39a4607406b4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4375de428e6b43e9aa3d276e4ce0017c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8d548e8e7c9f4c8e8fa996febeffeee1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/0aea2930f1484db48db41d03f38cb856] to archive 2024-11-11T22:27:20,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:27:20,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4aa74d0ebeee4181b04d746cc16bf9b0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4aa74d0ebeee4181b04d746cc16bf9b0 2024-11-11T22:27:20,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8bb6207d86694a7cb801a1829d0569b5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8bb6207d86694a7cb801a1829d0569b5 2024-11-11T22:27:20,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/99a5c3d80fc04b14b94b99985e383460 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/99a5c3d80fc04b14b94b99985e383460 2024-11-11T22:27:20,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/894705c39dc3434a80bf2c3143e59e40 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/894705c39dc3434a80bf2c3143e59e40 2024-11-11T22:27:20,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/bad7e3f37f5d40b38dc972ac2f49244e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/bad7e3f37f5d40b38dc972ac2f49244e 2024-11-11T22:27:20,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/b3689f75fb9e41f082a64a2d1adf2bfe to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/b3689f75fb9e41f082a64a2d1adf2bfe 2024-11-11T22:27:20,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/5f59e896a08c4da1b80cac09c6d41fea to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/5f59e896a08c4da1b80cac09c6d41fea 2024-11-11T22:27:20,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d98384b9fbd744d5a4c81de5a7b9a09c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d98384b9fbd744d5a4c81de5a7b9a09c 2024-11-11T22:27:20,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8e5ccc5798134eef988a8cd5aacdfea2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8e5ccc5798134eef988a8cd5aacdfea2 2024-11-11T22:27:20,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/6b27b8ac858d47d6888aaa5a7c6c69da to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/6b27b8ac858d47d6888aaa5a7c6c69da 2024-11-11T22:27:20,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/74a60f4cfd134915a9b76f41bd9a96cc to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/74a60f4cfd134915a9b76f41bd9a96cc 2024-11-11T22:27:20,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/50c4efb8ff7d44e3a280d7c21ebcd7b0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/50c4efb8ff7d44e3a280d7c21ebcd7b0 2024-11-11T22:27:20,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d483374391ed49b6ae78daacbdde29da to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d483374391ed49b6ae78daacbdde29da 2024-11-11T22:27:20,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/a9dd60fc761b4988b9e679ad539b470c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/a9dd60fc761b4988b9e679ad539b470c 2024-11-11T22:27:20,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/9018fe3853a34f77a0536fd9e84ddbd6 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/9018fe3853a34f77a0536fd9e84ddbd6 2024-11-11T22:27:20,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8b3710ba9fcd42c3ab4bc96d112a25ab to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8b3710ba9fcd42c3ab4bc96d112a25ab 2024-11-11T22:27:20,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d43e4f6d6967464cba5f8f0227fdcaef to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/d43e4f6d6967464cba5f8f0227fdcaef 2024-11-11T22:27:20,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/3595f97752e44a2c83dd39a4607406b4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/3595f97752e44a2c83dd39a4607406b4 2024-11-11T22:27:20,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4375de428e6b43e9aa3d276e4ce0017c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/4375de428e6b43e9aa3d276e4ce0017c 2024-11-11T22:27:20,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8d548e8e7c9f4c8e8fa996febeffeee1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8d548e8e7c9f4c8e8fa996febeffeee1 2024-11-11T22:27:20,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/0aea2930f1484db48db41d03f38cb856 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/0aea2930f1484db48db41d03f38cb856 2024-11-11T22:27:20,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1106e8f6207a493bbaf5ec40e77479ae, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/81c993ed7306439e81d65d472a605fd1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a474eb13ccd941ceaa6120b2753e7636, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/f789c26f6157442989a1007e3685bbe2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a13b7c0ada2a42f5b4c7b9e3a16f4c26, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/0c3226119686430786ea722857f3c180, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/27d1884487ca4c3c8b7400a21a238eba, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/6a2c3035b725495d9578475a930f2e78, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/4cfe29a42a304507a3810ab956438119, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/c577cab129564f0abf9b51318b9ead0d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/44f7e7d5f4cb4ee9a9f9b87bf8e69cbb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/aaaf817bdd704e01b47d3399d028de9c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/cf7779179e684394acb33e1f1b4a1a9a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/5e12258543f24da391a1d423bd7090a7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2146ce9960a34c7c96eeee6ac58252f1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/37edb9c31774408096cb5541e6e6bacc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/affc6482274a48e18e0ec8f8562a6a06, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/fca503f4ffe341ef8a29943350cde0bd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2b2e0c7042024357b2af42dc4745429b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2c3524f1be5e4f42a6e873c635bae20c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1b9db5049b59412eb663e5710efc8100] to archive 2024-11-11T22:27:20,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:27:20,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1106e8f6207a493bbaf5ec40e77479ae to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1106e8f6207a493bbaf5ec40e77479ae 2024-11-11T22:27:20,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/81c993ed7306439e81d65d472a605fd1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/81c993ed7306439e81d65d472a605fd1 2024-11-11T22:27:20,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a474eb13ccd941ceaa6120b2753e7636 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a474eb13ccd941ceaa6120b2753e7636 2024-11-11T22:27:20,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/f789c26f6157442989a1007e3685bbe2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/f789c26f6157442989a1007e3685bbe2 2024-11-11T22:27:20,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a13b7c0ada2a42f5b4c7b9e3a16f4c26 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/a13b7c0ada2a42f5b4c7b9e3a16f4c26 2024-11-11T22:27:20,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/0c3226119686430786ea722857f3c180 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/0c3226119686430786ea722857f3c180 2024-11-11T22:27:20,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/27d1884487ca4c3c8b7400a21a238eba to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/27d1884487ca4c3c8b7400a21a238eba 2024-11-11T22:27:20,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/6a2c3035b725495d9578475a930f2e78 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/6a2c3035b725495d9578475a930f2e78 2024-11-11T22:27:20,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/4cfe29a42a304507a3810ab956438119 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/4cfe29a42a304507a3810ab956438119 2024-11-11T22:27:20,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/c577cab129564f0abf9b51318b9ead0d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/c577cab129564f0abf9b51318b9ead0d 2024-11-11T22:27:20,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/44f7e7d5f4cb4ee9a9f9b87bf8e69cbb to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/44f7e7d5f4cb4ee9a9f9b87bf8e69cbb 2024-11-11T22:27:20,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/aaaf817bdd704e01b47d3399d028de9c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/aaaf817bdd704e01b47d3399d028de9c 2024-11-11T22:27:20,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/cf7779179e684394acb33e1f1b4a1a9a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/cf7779179e684394acb33e1f1b4a1a9a 2024-11-11T22:27:20,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/5e12258543f24da391a1d423bd7090a7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/5e12258543f24da391a1d423bd7090a7 2024-11-11T22:27:20,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2146ce9960a34c7c96eeee6ac58252f1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2146ce9960a34c7c96eeee6ac58252f1 2024-11-11T22:27:20,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/37edb9c31774408096cb5541e6e6bacc to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/37edb9c31774408096cb5541e6e6bacc 2024-11-11T22:27:20,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/affc6482274a48e18e0ec8f8562a6a06 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/affc6482274a48e18e0ec8f8562a6a06 2024-11-11T22:27:20,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/fca503f4ffe341ef8a29943350cde0bd to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/fca503f4ffe341ef8a29943350cde0bd 2024-11-11T22:27:20,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2b2e0c7042024357b2af42dc4745429b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2b2e0c7042024357b2af42dc4745429b 2024-11-11T22:27:20,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2c3524f1be5e4f42a6e873c635bae20c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/2c3524f1be5e4f42a6e873c635bae20c 2024-11-11T22:27:20,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1b9db5049b59412eb663e5710efc8100 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/1b9db5049b59412eb663e5710efc8100 2024-11-11T22:27:20,073 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=4 2024-11-11T22:27:20,073 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. 2024-11-11T22:27:20,074 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for 1077b11658f2ec3dcbfd684d096e1030: Waiting for close lock at 1731364038366Running coprocessor pre-close hooks at 1731364038366Disabling compacts and flushes for region at 1731364038366Disabling writes for close at 1731364038374 (+8 ms)Obtaining lock to block concurrent updates at 1731364038374Preparing flush snapshotting stores in 1077b11658f2ec3dcbfd684d096e1030 at 1731364038374Finished memstore snapshotting TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731364038374Flushing stores of TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030. at 1731364038374Flushing 1077b11658f2ec3dcbfd684d096e1030/A: creating writer at 1731364038374Flushing mob file 1077b11658f2ec3dcbfd684d096e1030/A: appending metadata at 1731364038378 (+4 ms)Flushing mob file 1077b11658f2ec3dcbfd684d096e1030/A: closing flushed file at 1731364038378Flushing 1077b11658f2ec3dcbfd684d096e1030/A: appending metadata at 1731364038785 (+407 ms)Flushing 1077b11658f2ec3dcbfd684d096e1030/A: closing flushed file at 1731364038785Flushing 1077b11658f2ec3dcbfd684d096e1030/B: creating writer at 1731364039192 (+407 ms)Flushing 1077b11658f2ec3dcbfd684d096e1030/B: appending metadata at 1731364039195 (+3 ms)Flushing 1077b11658f2ec3dcbfd684d096e1030/B: closing flushed file at 1731364039195Flushing 1077b11658f2ec3dcbfd684d096e1030/C: creating writer at 1731364039602 (+407 ms)Flushing 1077b11658f2ec3dcbfd684d096e1030/C: appending metadata at 1731364039604 (+2 ms)Flushing 1077b11658f2ec3dcbfd684d096e1030/C: closing flushed file at 1731364039604Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a6a35bf: reopening flushed file at 1731364040010 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fb753a3: reopening flushed file at 1731364040014 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51d5b6c2: reopening flushed file at 1731364040017 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 1077b11658f2ec3dcbfd684d096e1030 in 1646ms, sequenceid=334, compaction requested=true at 1731364040021 (+4 ms)Writing region close event to WAL at 1731364040070 (+49 ms)Running coprocessor post-close hooks at 1731364040073 (+3 ms)Closed at 1731364040073 2024-11-11T22:27:20,075 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed 1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,076 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=1077b11658f2ec3dcbfd684d096e1030, regionState=CLOSED 2024-11-11T22:27:20,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:27:20,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T22:27:20,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure 1077b11658f2ec3dcbfd684d096e1030, server=8d82b89dd27d,39613,1731363918313 in 1.8640 sec 2024-11-11T22:27:20,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-11T22:27:20,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1077b11658f2ec3dcbfd684d096e1030, UNASSIGN in 1.8680 sec 2024-11-11T22:27:20,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T22:27:20,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8710 sec 2024-11-11T22:27:20,083 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364040083"}]},"ts":"1731364040083"} 2024-11-11T22:27:20,084 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:27:20,084 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:27:20,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8790 sec 2024-11-11T22:27:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:27:20,353 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:20,354 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:27:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:20,355 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T22:27:20,356 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:20,358 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,360 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/recovered.edits] 2024-11-11T22:27:20,362 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/12ee55b9553c47ceb36c82b78d8aebae to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/12ee55b9553c47ceb36c82b78d8aebae 2024-11-11T22:27:20,363 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/698a1f8c872b4fd990fc7a46260c13fe to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/698a1f8c872b4fd990fc7a46260c13fe 2024-11-11T22:27:20,364 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/c7045b29a2c14c28b49bd2d26d7a67ff to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/A/c7045b29a2c14c28b49bd2d26d7a67ff 2024-11-11T22:27:20,365 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8e668d656efb44c0a24df49950f51c22 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/8e668d656efb44c0a24df49950f51c22 2024-11-11T22:27:20,366 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/a6fb66de146443d6abc97c14c4b0cd99 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/a6fb66de146443d6abc97c14c4b0cd99 2024-11-11T22:27:20,367 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/f211498b770645db842ab6c95a39907f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/B/f211498b770645db842ab6c95a39907f 2024-11-11T22:27:20,369 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/3d0c2cfdc6904e69b50c8d5bc3f5fd45 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/3d0c2cfdc6904e69b50c8d5bc3f5fd45 2024-11-11T22:27:20,369 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/637d708fd69544e69eeca003607d44ff to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/637d708fd69544e69eeca003607d44ff 2024-11-11T22:27:20,370 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/fca37aef6f6046d5a69cf12d529f7786 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/C/fca37aef6f6046d5a69cf12d529f7786 2024-11-11T22:27:20,372 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/recovered.edits/337.seqid to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030/recovered.edits/337.seqid 2024-11-11T22:27:20,372 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,372 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:27:20,373 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:27:20,373 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T22:27:20,376 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110dc565935e0f4094b297574cfce6dd63_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110dc565935e0f4094b297574cfce6dd63_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,377 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111112c9c82129854718b28778ca9a060692_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111112c9c82129854718b28778ca9a060692_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,377 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111bc16c7af03348b9a087d97276e0e759_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111bc16c7af03348b9a087d97276e0e759_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,378 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111faa7da295b94d93932ad776e2af17e4_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111faa7da295b94d93932ad776e2af17e4_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,379 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111291d6c05d6194497937641628d583022_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111291d6c05d6194497937641628d583022_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,380 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114eba8ae9831f40d880cc178865b9e4f6_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114eba8ae9831f40d880cc178865b9e4f6_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,380 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154bb513ed63f476bb22db6971b5a6cc2_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154bb513ed63f476bb22db6971b5a6cc2_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,381 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f7eebbaea894002b5d078a7791db4b7_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f7eebbaea894002b5d078a7791db4b7_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,382 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117a62698f2d5947bda75890853f6c700a_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117a62698f2d5947bda75890853f6c700a_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,382 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119385893e0b1d47d09a8f679c243c3ec1_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119385893e0b1d47d09a8f679c243c3ec1_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,383 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111198f4a7e497d843e5a85219f78b165905_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111198f4a7e497d843e5a85219f78b165905_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,384 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cc17efb3c7ee4b6da413ad4a848e9be1_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cc17efb3c7ee4b6da413ad4a848e9be1_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,384 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d037afbd00b74c5c99f2b33944684476_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d037afbd00b74c5c99f2b33944684476_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,385 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dc78278c06804fd398e856208690d384_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dc78278c06804fd398e856208690d384_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,386 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dd6b4cd39571448fae33db851ba6e400_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dd6b4cd39571448fae33db851ba6e400_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,387 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ead40c3e95aa49f3a9e052261670af55_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ead40c3e95aa49f3a9e052261670af55_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,388 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111eb58517a6deb4d069727fc81c25311a6_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111eb58517a6deb4d069727fc81c25311a6_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,388 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fca5e5929787423b8d6432ca33086aa1_1077b11658f2ec3dcbfd684d096e1030 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fca5e5929787423b8d6432ca33086aa1_1077b11658f2ec3dcbfd684d096e1030 2024-11-11T22:27:20,389 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:27:20,390 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:20,392 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:27:20,393 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:27:20,394 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:20,394 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:27:20,395 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731364040394"}]},"ts":"9223372036854775807"} 2024-11-11T22:27:20,396 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:27:20,396 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1077b11658f2ec3dcbfd684d096e1030, NAME => 'TestAcidGuarantees,,1731364013405.1077b11658f2ec3dcbfd684d096e1030.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:27:20,396 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:27:20,396 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731364040396"}]},"ts":"9223372036854775807"} 2024-11-11T22:27:20,397 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:27:20,398 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:20,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-11T22:27:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T22:27:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:27:20,464 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:20,472 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 240) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-97114466_22 at /127.0.0.1:49252 [Waiting for operation #920] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-97114466_22 at /127.0.0.1:58086 [Waiting for operation #420] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_437819696_22 at /127.0.0.1:58096 [Waiting for operation #424] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_437819696_22 at /127.0.0.1:58094 [Waiting for operation #402] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=475 (was 512), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=5462 (was 5583) 2024-11-11T22:27:20,480 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=475, ProcessCount=11, AvailableMemoryMB=5461 2024-11-11T22:27:20,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:27:20,481 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:27:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:20,483 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:27:20,483 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:20,483 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-11T22:27:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T22:27:20,484 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:27:20,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742254_1430 (size=959) 2024-11-11T22:27:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T22:27:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T22:27:20,890 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e 2024-11-11T22:27:20,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742255_1431 (size=53) 2024-11-11T22:27:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T22:27:21,298 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:27:21,298 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 324e6f269deb5f32d638f5ef06ba3bab, disabling compactions & flushes 2024-11-11T22:27:21,298 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:21,298 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:21,298 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. after waiting 0 ms 2024-11-11T22:27:21,298 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:21,298 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:21,298 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 324e6f269deb5f32d638f5ef06ba3bab: Waiting for close lock at 1731364041298Disabling compacts and flushes for region at 1731364041298Disabling writes for close at 1731364041298Writing region close event to WAL at 1731364041298Closed at 1731364041298 2024-11-11T22:27:21,299 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:27:21,300 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731364041299"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731364041299"}]},"ts":"1731364041299"} 2024-11-11T22:27:21,301 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:27:21,301 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:27:21,301 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364041301"}]},"ts":"1731364041301"} 2024-11-11T22:27:21,302 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:27:21,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=324e6f269deb5f32d638f5ef06ba3bab, ASSIGN}] 2024-11-11T22:27:21,303 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=324e6f269deb5f32d638f5ef06ba3bab, ASSIGN 2024-11-11T22:27:21,304 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=324e6f269deb5f32d638f5ef06ba3bab, ASSIGN; state=OFFLINE, location=8d82b89dd27d,39613,1731363918313; forceNewPlan=false, retain=false 2024-11-11T22:27:21,454 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=324e6f269deb5f32d638f5ef06ba3bab, regionState=OPENING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:21,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=324e6f269deb5f32d638f5ef06ba3bab, ASSIGN because future has completed 2024-11-11T22:27:21,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:27:21,487 WARN [regionserver/8d82b89dd27d:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-11T22:27:21,610 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:21,610 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:27:21,611 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,611 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:27:21,611 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,611 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,612 INFO [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,613 INFO [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 324e6f269deb5f32d638f5ef06ba3bab columnFamilyName A 2024-11-11T22:27:21,613 DEBUG [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:21,614 INFO [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] regionserver.HStore(327): Store=324e6f269deb5f32d638f5ef06ba3bab/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:27:21,614 INFO [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,615 INFO [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 324e6f269deb5f32d638f5ef06ba3bab columnFamilyName B 2024-11-11T22:27:21,615 DEBUG [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:21,615 INFO [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] regionserver.HStore(327): Store=324e6f269deb5f32d638f5ef06ba3bab/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:27:21,615 INFO [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,616 INFO [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 324e6f269deb5f32d638f5ef06ba3bab columnFamilyName C 2024-11-11T22:27:21,616 DEBUG [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:21,616 INFO [StoreOpener-324e6f269deb5f32d638f5ef06ba3bab-1 {}] regionserver.HStore(327): Store=324e6f269deb5f32d638f5ef06ba3bab/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:27:21,616 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,617 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,617 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,617 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,617 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,618 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:27:21,619 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,620 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:27:21,620 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened 324e6f269deb5f32d638f5ef06ba3bab; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59187778, jitterRate=-0.11803337931632996}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:27:21,620 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:21,621 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for 324e6f269deb5f32d638f5ef06ba3bab: Running coprocessor pre-open hook at 1731364041611Writing region info on filesystem at 1731364041611Initializing all the Stores at 1731364041612 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364041612Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364041612Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364041612Cleaning up temporary data from old regions at 1731364041617 (+5 ms)Running coprocessor post-open hooks at 1731364041620 (+3 ms)Region opened successfully at 1731364041621 (+1 ms) 2024-11-11T22:27:21,621 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., pid=110, masterSystemTime=1731364041608 2024-11-11T22:27:21,623 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:21,623 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T22:27:21,623 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=324e6f269deb5f32d638f5ef06ba3bab, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:21,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:27:21,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T22:27:21,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure 324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 in 169 msec 2024-11-11T22:27:21,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-11T22:27:21,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=324e6f269deb5f32d638f5ef06ba3bab, ASSIGN in 323 msec 2024-11-11T22:27:21,628 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:27:21,628 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364041628"}]},"ts":"1731364041628"} 2024-11-11T22:27:21,629 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:27:21,630 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:27:21,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-11T22:27:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-11T22:27:22,633 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:22,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,635 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7014110e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,636 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c54b14d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,637 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,639 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:27:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,40133,1731363917560 2024-11-11T22:27:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f31ea91 2024-11-11T22:27:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:27:22,641 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:27:22,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,643 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c7e12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,644 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cd560e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,646 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2824b841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,647 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@498878fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,649 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9349ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,650 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38cb697e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,651 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2138dc82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,652 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478ba428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cdfde48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,655 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb10ee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,657 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48443120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,658 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b23885e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,660 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13635cea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,661 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51702b07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,663 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc7aace, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,664 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769ed060, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,669 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c1a75d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,670 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f29a3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:22,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:22,673 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e40c52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:22,674 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594ef208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:22,675 DEBUG [Thread-1952 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,675 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,675 DEBUG [Thread-1956 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,676 DEBUG [Thread-1941 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:27:22,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:27:22,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:27:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:27:22,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:22,680 DEBUG [Thread-1960 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-11T22:27:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,681 DEBUG [Thread-1949 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,682 DEBUG [Thread-1947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T22:27:22,683 DEBUG [Thread-1947 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,683 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:22,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,683 DEBUG [Thread-1947 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,684 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:22,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:22,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:27:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:27:22,688 DEBUG [Thread-1947 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:27:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:22,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:27:22,691 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,691 DEBUG [Thread-1958 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,694 DEBUG [Thread-1954 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:22,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:27:22,695 DEBUG [Thread-1954 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:27:22,695 DEBUG [Thread-1954 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:22,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2] 2024-11-11T22:27:22,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/2b9fc003359645cea05d370b423498d4 is 50, key is test_row_0/A:col10/1731364042686/Put/seqid=0 2024-11-11T22:27:22,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364102724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:22,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364102724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:22,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364102730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:22,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:22,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742256_1432 (size=19021) 2024-11-11T22:27:22,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/2b9fc003359645cea05d370b423498d4 2024-11-11T22:27:22,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364102732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:22,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:22,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364102734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:22,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/7ff4033914bf40fba9947481cf03c60f is 50, key is test_row_0/B:col10/1731364042686/Put/seqid=0 2024-11-11T22:27:22,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742257_1433 (size=12001) 2024-11-11T22:27:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T22:27:22,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T22:27:22,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:22,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:22,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:22,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:22,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364102843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364102843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364102843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364102843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:22,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364102853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:22,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T22:27:22,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:22,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:22,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:22,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:22,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T22:27:23,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364103053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364103053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364103053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364103053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364103063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T22:27:23,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:23,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:23,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:23,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:23,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:23,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/7ff4033914bf40fba9947481cf03c60f 2024-11-11T22:27:23,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/789aece9d5da4861a477e748e6a41038 is 50, key is test_row_0/C:col10/1731364042686/Put/seqid=0 2024-11-11T22:27:23,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742258_1434 (size=12001) 2024-11-11T22:27:23,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/789aece9d5da4861a477e748e6a41038 2024-11-11T22:27:23,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/2b9fc003359645cea05d370b423498d4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2b9fc003359645cea05d370b423498d4 2024-11-11T22:27:23,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2b9fc003359645cea05d370b423498d4, entries=300, sequenceid=12, filesize=18.6 K 2024-11-11T22:27:23,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/7ff4033914bf40fba9947481cf03c60f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/7ff4033914bf40fba9947481cf03c60f 2024-11-11T22:27:23,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/7ff4033914bf40fba9947481cf03c60f, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T22:27:23,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/789aece9d5da4861a477e748e6a41038 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/789aece9d5da4861a477e748e6a41038 2024-11-11T22:27:23,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/789aece9d5da4861a477e748e6a41038, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T22:27:23,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 324e6f269deb5f32d638f5ef06ba3bab in 556ms, sequenceid=12, compaction requested=false 2024-11-11T22:27:23,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:23,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T22:27:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:23,295 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:27:23,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/278f0ccc3c5741529cd1d18683e1ae5b is 50, key is test_row_0/A:col10/1731364042729/Put/seqid=0 2024-11-11T22:27:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T22:27:23,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742259_1435 (size=12001) 2024-11-11T22:27:23,318 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/278f0ccc3c5741529cd1d18683e1ae5b 2024-11-11T22:27:23,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6bc55e1a90c44c0396e52ea1349b455f is 50, key is test_row_0/B:col10/1731364042729/Put/seqid=0 2024-11-11T22:27:23,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742260_1436 (size=12001) 2024-11-11T22:27:23,346 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6bc55e1a90c44c0396e52ea1349b455f 2024-11-11T22:27:23,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/d77fa2f41cc043b0a8c51b623340a88b is 50, key is test_row_0/C:col10/1731364042729/Put/seqid=0 2024-11-11T22:27:23,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742261_1437 (size=12001) 2024-11-11T22:27:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:23,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:23,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364103372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364103373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364103374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364103374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364103375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364103483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364103493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364103493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364103494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364103693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364103703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364103703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364103704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:23,762 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/d77fa2f41cc043b0a8c51b623340a88b 2024-11-11T22:27:23,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/278f0ccc3c5741529cd1d18683e1ae5b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/278f0ccc3c5741529cd1d18683e1ae5b 2024-11-11T22:27:23,774 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/278f0ccc3c5741529cd1d18683e1ae5b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T22:27:23,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6bc55e1a90c44c0396e52ea1349b455f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bc55e1a90c44c0396e52ea1349b455f 2024-11-11T22:27:23,778 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bc55e1a90c44c0396e52ea1349b455f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T22:27:23,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/d77fa2f41cc043b0a8c51b623340a88b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/d77fa2f41cc043b0a8c51b623340a88b 2024-11-11T22:27:23,782 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/d77fa2f41cc043b0a8c51b623340a88b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T22:27:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T22:27:23,783 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 324e6f269deb5f32d638f5ef06ba3bab in 489ms, sequenceid=37, compaction requested=false 2024-11-11T22:27:23,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:23,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:23,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-11T22:27:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-11T22:27:23,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T22:27:23,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1000 sec 2024-11-11T22:27:23,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.1060 sec 2024-11-11T22:27:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T22:27:23,823 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:23,824 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-11T22:27:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T22:27:23,831 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:23,832 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:23,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:23,895 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:23,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:27:23,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/c3092a9f81ec484fbb5d36185060ff25 is 50, key is test_row_0/A:col10/1731364043894/Put/seqid=0 2024-11-11T22:27:23,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742262_1438 (size=12001) 2024-11-11T22:27:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T22:27:23,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:23,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:23,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364104013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364104013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364104013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364104022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364104022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:24,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:24,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T22:27:24,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364104144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364104144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/c3092a9f81ec484fbb5d36185060ff25 2024-11-11T22:27:24,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/54bef576116241b8bee3c3036fd2c074 is 50, key is test_row_0/B:col10/1731364043894/Put/seqid=0 2024-11-11T22:27:24,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742263_1439 (size=12001) 2024-11-11T22:27:24,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364104353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364104363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T22:27:24,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364104533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364104533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364104533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364104663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364104673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:24,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/54bef576116241b8bee3c3036fd2c074 2024-11-11T22:27:24,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/f036c046b398414282952ff85ac312cb is 50, key is test_row_0/C:col10/1731364043894/Put/seqid=0 2024-11-11T22:27:24,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742264_1440 (size=12001) 2024-11-11T22:27:24,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:24,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:24,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:24,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:24,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:24,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T22:27:25,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:25,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:25,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/f036c046b398414282952ff85ac312cb 2024-11-11T22:27:25,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/c3092a9f81ec484fbb5d36185060ff25 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c3092a9f81ec484fbb5d36185060ff25 2024-11-11T22:27:25,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c3092a9f81ec484fbb5d36185060ff25, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T22:27:25,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/54bef576116241b8bee3c3036fd2c074 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/54bef576116241b8bee3c3036fd2c074 2024-11-11T22:27:25,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/54bef576116241b8bee3c3036fd2c074, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T22:27:25,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/f036c046b398414282952ff85ac312cb as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f036c046b398414282952ff85ac312cb 2024-11-11T22:27:25,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f036c046b398414282952ff85ac312cb, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T22:27:25,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 324e6f269deb5f32d638f5ef06ba3bab in 1271ms, sequenceid=49, compaction requested=true 2024-11-11T22:27:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:25,166 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:25,166 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:25,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:25,167 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:25,167 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:25,167 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/B is initiating minor compaction (all files) 2024-11-11T22:27:25,167 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/A is initiating minor compaction (all files) 2024-11-11T22:27:25,167 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/A in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,167 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/B in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,167 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/7ff4033914bf40fba9947481cf03c60f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bc55e1a90c44c0396e52ea1349b455f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/54bef576116241b8bee3c3036fd2c074] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=35.2 K 2024-11-11T22:27:25,167 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2b9fc003359645cea05d370b423498d4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/278f0ccc3c5741529cd1d18683e1ae5b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c3092a9f81ec484fbb5d36185060ff25] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=42.0 K 2024-11-11T22:27:25,167 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b9fc003359645cea05d370b423498d4, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731364042679 2024-11-11T22:27:25,167 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ff4033914bf40fba9947481cf03c60f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731364042686 2024-11-11T22:27:25,168 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 278f0ccc3c5741529cd1d18683e1ae5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731364042719 2024-11-11T22:27:25,168 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3092a9f81ec484fbb5d36185060ff25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731364043373 2024-11-11T22:27:25,173 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc55e1a90c44c0396e52ea1349b455f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731364042719 2024-11-11T22:27:25,174 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 54bef576116241b8bee3c3036fd2c074, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731364043373 2024-11-11T22:27:25,176 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#A#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:25,176 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/a4a63cddc1bf4df58eaeb4b02f9a4b21 is 50, key is test_row_0/A:col10/1731364043894/Put/seqid=0 2024-11-11T22:27:25,184 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#B#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:25,185 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/ceb4841ba7ef4252bbc5e0c10d4219d5 is 50, key is test_row_0/B:col10/1731364043894/Put/seqid=0 2024-11-11T22:27:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:25,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742265_1441 (size=12104) 2024-11-11T22:27:25,191 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:25,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:27:25,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742266_1442 (size=12104) 2024-11-11T22:27:25,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/590e8a2c94d34ab8a8506d93e44fddc7 is 50, key is test_row_0/A:col10/1731364044011/Put/seqid=0 2024-11-11T22:27:25,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742267_1443 (size=14341) 2024-11-11T22:27:25,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/590e8a2c94d34ab8a8506d93e44fddc7 2024-11-11T22:27:25,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/f3e2f990714b4a099aba86516adecf29 is 50, key is test_row_0/B:col10/1731364044011/Put/seqid=0 2024-11-11T22:27:25,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:25,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:25,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742268_1444 (size=12001) 2024-11-11T22:27:25,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/f3e2f990714b4a099aba86516adecf29 2024-11-11T22:27:25,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364105227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/a00fba3c06ba4ab4a84d9e8b30cdf84b is 50, key is test_row_0/C:col10/1731364044011/Put/seqid=0 2024-11-11T22:27:25,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364105228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742269_1445 (size=12001) 2024-11-11T22:27:25,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364105343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364105343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:25,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:25,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:25,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364105543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,548 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364105553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364105553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,557 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364105553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,558 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364105563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,598 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/a4a63cddc1bf4df58eaeb4b02f9a4b21 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a4a63cddc1bf4df58eaeb4b02f9a4b21 2024-11-11T22:27:25,602 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:27:25,603 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/ceb4841ba7ef4252bbc5e0c10d4219d5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ceb4841ba7ef4252bbc5e0c10d4219d5 2024-11-11T22:27:25,605 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/A of 324e6f269deb5f32d638f5ef06ba3bab into a4a63cddc1bf4df58eaeb4b02f9a4b21(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:25,605 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:25,605 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/A, priority=13, startTime=1731364045166; duration=0sec 2024-11-11T22:27:25,605 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:25,605 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:A 2024-11-11T22:27:25,605 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:25,607 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:25,607 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/C is initiating minor compaction (all files) 2024-11-11T22:27:25,607 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/C in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,607 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/789aece9d5da4861a477e748e6a41038, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/d77fa2f41cc043b0a8c51b623340a88b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f036c046b398414282952ff85ac312cb] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=35.2 K 2024-11-11T22:27:25,608 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 789aece9d5da4861a477e748e6a41038, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731364042686 2024-11-11T22:27:25,608 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting d77fa2f41cc043b0a8c51b623340a88b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731364042719 2024-11-11T22:27:25,609 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting f036c046b398414282952ff85ac312cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731364043373 2024-11-11T22:27:25,610 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/B of 324e6f269deb5f32d638f5ef06ba3bab into ceb4841ba7ef4252bbc5e0c10d4219d5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:25,610 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:25,610 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/B, priority=13, startTime=1731364045166; duration=0sec 2024-11-11T22:27:25,610 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:25,610 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:B 2024-11-11T22:27:25,618 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:25,618 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/96b0a2c978f845859c1801e11ea2770b is 50, key is test_row_0/C:col10/1731364043894/Put/seqid=0 2024-11-11T22:27:25,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742270_1446 (size=12104) 2024-11-11T22:27:25,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/a00fba3c06ba4ab4a84d9e8b30cdf84b 2024-11-11T22:27:25,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/590e8a2c94d34ab8a8506d93e44fddc7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/590e8a2c94d34ab8a8506d93e44fddc7 2024-11-11T22:27:25,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/590e8a2c94d34ab8a8506d93e44fddc7, entries=200, sequenceid=74, filesize=14.0 K 2024-11-11T22:27:25,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/f3e2f990714b4a099aba86516adecf29 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/f3e2f990714b4a099aba86516adecf29 2024-11-11T22:27:25,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/f3e2f990714b4a099aba86516adecf29, entries=150, sequenceid=74, filesize=11.7 K 2024-11-11T22:27:25,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/a00fba3c06ba4ab4a84d9e8b30cdf84b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a00fba3c06ba4ab4a84d9e8b30cdf84b 2024-11-11T22:27:25,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a00fba3c06ba4ab4a84d9e8b30cdf84b, entries=150, sequenceid=74, filesize=11.7 K 2024-11-11T22:27:25,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 324e6f269deb5f32d638f5ef06ba3bab in 469ms, sequenceid=74, compaction requested=false 2024-11-11T22:27:25,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:25,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T22:27:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:25,672 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:27:25,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/27399c6de50547afaf1840e986c7aa8e is 50, key is test_row_0/A:col10/1731364045225/Put/seqid=0 2024-11-11T22:27:25,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742271_1447 (size=12001) 2024-11-11T22:27:25,681 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/27399c6de50547afaf1840e986c7aa8e 2024-11-11T22:27:25,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/25be6501562f41d996baa3efdb1bdb34 is 50, key is test_row_0/B:col10/1731364045225/Put/seqid=0 2024-11-11T22:27:25,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742272_1448 (size=12001) 2024-11-11T22:27:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:25,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364105956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364105960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T22:27:26,033 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/96b0a2c978f845859c1801e11ea2770b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/96b0a2c978f845859c1801e11ea2770b 2024-11-11T22:27:26,037 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/C of 324e6f269deb5f32d638f5ef06ba3bab into 96b0a2c978f845859c1801e11ea2770b(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:26,037 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:26,037 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/C, priority=13, startTime=1731364045166; duration=0sec 2024-11-11T22:27:26,037 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:26,037 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:C 2024-11-11T22:27:26,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364106063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:26,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364106073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:26,091 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/25be6501562f41d996baa3efdb1bdb34 2024-11-11T22:27:26,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/c499138e11ba4f9d9005f9dafbabaec5 is 50, key is test_row_0/C:col10/1731364045225/Put/seqid=0 2024-11-11T22:27:26,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742273_1449 (size=12001) 2024-11-11T22:27:26,106 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/c499138e11ba4f9d9005f9dafbabaec5 2024-11-11T22:27:26,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/27399c6de50547afaf1840e986c7aa8e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/27399c6de50547afaf1840e986c7aa8e 2024-11-11T22:27:26,116 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/27399c6de50547afaf1840e986c7aa8e, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T22:27:26,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/25be6501562f41d996baa3efdb1bdb34 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/25be6501562f41d996baa3efdb1bdb34 2024-11-11T22:27:26,119 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/25be6501562f41d996baa3efdb1bdb34, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T22:27:26,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/c499138e11ba4f9d9005f9dafbabaec5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c499138e11ba4f9d9005f9dafbabaec5 2024-11-11T22:27:26,126 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c499138e11ba4f9d9005f9dafbabaec5, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T22:27:26,126 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 324e6f269deb5f32d638f5ef06ba3bab in 454ms, sequenceid=87, compaction requested=true 2024-11-11T22:27:26,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:26,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:26,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-11T22:27:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-11T22:27:26,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T22:27:26,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2960 sec 2024-11-11T22:27:26,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.3060 sec 2024-11-11T22:27:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:26,284 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:26,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T22:27:26,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/23d72ebd4c58404a83213daf68299b3e is 50, key is test_row_0/A:col10/1731364046283/Put/seqid=0 2024-11-11T22:27:26,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742274_1450 (size=14341) 2024-11-11T22:27:26,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/23d72ebd4c58404a83213daf68299b3e 2024-11-11T22:27:26,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/9625a547c5c94dcd85b2c72b3b3d9b17 is 50, key is test_row_0/B:col10/1731364046283/Put/seqid=0 2024-11-11T22:27:26,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742275_1451 (size=12001) 2024-11-11T22:27:26,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364106318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:26,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364106319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:26,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364106433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364106433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:26,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364106643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364106643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:26,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/9625a547c5c94dcd85b2c72b3b3d9b17 2024-11-11T22:27:26,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/f449b8f6bf99424fad788c9fc230e402 is 50, key is test_row_0/C:col10/1731364046283/Put/seqid=0 2024-11-11T22:27:26,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742276_1452 (size=12001) 2024-11-11T22:27:26,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364106953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:26,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364106953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:27,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/f449b8f6bf99424fad788c9fc230e402 2024-11-11T22:27:27,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/23d72ebd4c58404a83213daf68299b3e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/23d72ebd4c58404a83213daf68299b3e 2024-11-11T22:27:27,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/23d72ebd4c58404a83213daf68299b3e, entries=200, sequenceid=114, filesize=14.0 K 2024-11-11T22:27:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/9625a547c5c94dcd85b2c72b3b3d9b17 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/9625a547c5c94dcd85b2c72b3b3d9b17 2024-11-11T22:27:27,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/9625a547c5c94dcd85b2c72b3b3d9b17, entries=150, sequenceid=114, filesize=11.7 K 2024-11-11T22:27:27,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/f449b8f6bf99424fad788c9fc230e402 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f449b8f6bf99424fad788c9fc230e402 2024-11-11T22:27:27,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f449b8f6bf99424fad788c9fc230e402, entries=150, sequenceid=114, filesize=11.7 K 2024-11-11T22:27:27,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 324e6f269deb5f32d638f5ef06ba3bab in 844ms, sequenceid=114, compaction requested=true 2024-11-11T22:27:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:27,128 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:27,128 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:27,129 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:27,129 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/B is initiating minor compaction (all files) 2024-11-11T22:27:27,129 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:27,129 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/B in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:27,129 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/A is initiating minor compaction (all files) 2024-11-11T22:27:27,129 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/A in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:27,129 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ceb4841ba7ef4252bbc5e0c10d4219d5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/f3e2f990714b4a099aba86516adecf29, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/25be6501562f41d996baa3efdb1bdb34, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/9625a547c5c94dcd85b2c72b3b3d9b17] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=47.0 K 2024-11-11T22:27:27,130 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a4a63cddc1bf4df58eaeb4b02f9a4b21, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/590e8a2c94d34ab8a8506d93e44fddc7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/27399c6de50547afaf1840e986c7aa8e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/23d72ebd4c58404a83213daf68299b3e] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=51.5 K 2024-11-11T22:27:27,130 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a63cddc1bf4df58eaeb4b02f9a4b21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731364043373 2024-11-11T22:27:27,130 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting ceb4841ba7ef4252bbc5e0c10d4219d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731364043373 2024-11-11T22:27:27,130 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f3e2f990714b4a099aba86516adecf29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731364044010 2024-11-11T22:27:27,130 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 590e8a2c94d34ab8a8506d93e44fddc7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731364044010 2024-11-11T22:27:27,130 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 25be6501562f41d996baa3efdb1bdb34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731364045200 2024-11-11T22:27:27,130 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27399c6de50547afaf1840e986c7aa8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731364045200 2024-11-11T22:27:27,131 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 9625a547c5c94dcd85b2c72b3b3d9b17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731364045949 2024-11-11T22:27:27,131 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23d72ebd4c58404a83213daf68299b3e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731364045949 2024-11-11T22:27:27,138 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#B#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:27,138 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#A#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:27,139 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/d44353d4adc949069bc4736095c390a0 is 50, key is test_row_0/A:col10/1731364046283/Put/seqid=0 2024-11-11T22:27:27,139 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/45378cf341b6446598c5eda1c897a957 is 50, key is test_row_0/B:col10/1731364046283/Put/seqid=0 2024-11-11T22:27:27,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742277_1453 (size=12241) 2024-11-11T22:27:27,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742278_1454 (size=12241) 2024-11-11T22:27:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:27,466 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:27,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:27:27,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/adc59025ec6b4572ae3427fb3993b3be is 50, key is test_row_0/A:col10/1731364046317/Put/seqid=0 2024-11-11T22:27:27,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742279_1455 (size=14341) 2024-11-11T22:27:27,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/adc59025ec6b4572ae3427fb3993b3be 2024-11-11T22:27:27,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/3fa60c18f2524a27a8608f6cfe622b1e is 50, key is test_row_0/B:col10/1731364046317/Put/seqid=0 2024-11-11T22:27:27,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742280_1456 (size=12001) 2024-11-11T22:27:27,546 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/45378cf341b6446598c5eda1c897a957 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/45378cf341b6446598c5eda1c897a957 2024-11-11T22:27:27,546 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/d44353d4adc949069bc4736095c390a0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/d44353d4adc949069bc4736095c390a0 2024-11-11T22:27:27,551 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/B of 324e6f269deb5f32d638f5ef06ba3bab into 45378cf341b6446598c5eda1c897a957(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:27,551 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:27,551 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/B, priority=12, startTime=1731364047128; duration=0sec 2024-11-11T22:27:27,552 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:27,552 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:B 2024-11-11T22:27:27,552 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:27,554 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:27,554 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/C is initiating minor compaction (all files) 2024-11-11T22:27:27,554 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/C in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:27,554 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/96b0a2c978f845859c1801e11ea2770b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a00fba3c06ba4ab4a84d9e8b30cdf84b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c499138e11ba4f9d9005f9dafbabaec5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f449b8f6bf99424fad788c9fc230e402] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=47.0 K 2024-11-11T22:27:27,554 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b0a2c978f845859c1801e11ea2770b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731364043373 2024-11-11T22:27:27,555 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a00fba3c06ba4ab4a84d9e8b30cdf84b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731364044010 2024-11-11T22:27:27,555 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting c499138e11ba4f9d9005f9dafbabaec5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731364045200 2024-11-11T22:27:27,558 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f449b8f6bf99424fad788c9fc230e402, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731364045949 2024-11-11T22:27:27,558 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/A of 324e6f269deb5f32d638f5ef06ba3bab into d44353d4adc949069bc4736095c390a0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:27,558 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:27,559 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/A, priority=12, startTime=1731364047128; duration=0sec 2024-11-11T22:27:27,559 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:27,559 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:A 2024-11-11T22:27:27,575 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#C#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:27,576 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/dd169a2958e046f68e352494782b8d0b is 50, key is test_row_0/C:col10/1731364046283/Put/seqid=0 2024-11-11T22:27:27,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742281_1457 (size=12241) 2024-11-11T22:27:27,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364107572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364107572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364107573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:27,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364107573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:27,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:27,587 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364107573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:27,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:27,588 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:27,588 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:27,593 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/dd169a2958e046f68e352494782b8d0b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dd169a2958e046f68e352494782b8d0b 2024-11-11T22:27:27,597 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/C of 324e6f269deb5f32d638f5ef06ba3bab into dd169a2958e046f68e352494782b8d0b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:27,598 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:27,598 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/C, priority=12, startTime=1731364047128; duration=0sec 2024-11-11T22:27:27,598 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:27,598 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:C 2024-11-11T22:27:27,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364107693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364107693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:27,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/3fa60c18f2524a27a8608f6cfe622b1e 2024-11-11T22:27:27,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/08410c00ea254ab4b86be4fadc0abcf6 is 50, key is test_row_0/C:col10/1731364046317/Put/seqid=0 2024-11-11T22:27:27,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742282_1458 (size=12001) 2024-11-11T22:27:27,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/08410c00ea254ab4b86be4fadc0abcf6 2024-11-11T22:27:27,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/adc59025ec6b4572ae3427fb3993b3be as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/adc59025ec6b4572ae3427fb3993b3be 2024-11-11T22:27:27,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/adc59025ec6b4572ae3427fb3993b3be, entries=200, sequenceid=126, filesize=14.0 K 2024-11-11T22:27:27,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364107903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:27,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364107903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:27,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/3fa60c18f2524a27a8608f6cfe622b1e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3fa60c18f2524a27a8608f6cfe622b1e 2024-11-11T22:27:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:27,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3fa60c18f2524a27a8608f6cfe622b1e, entries=150, sequenceid=126, filesize=11.7 K 2024-11-11T22:27:27,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/08410c00ea254ab4b86be4fadc0abcf6 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/08410c00ea254ab4b86be4fadc0abcf6 2024-11-11T22:27:27,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/08410c00ea254ab4b86be4fadc0abcf6, entries=150, sequenceid=126, filesize=11.7 K 2024-11-11T22:27:27,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 324e6f269deb5f32d638f5ef06ba3bab in 448ms, sequenceid=126, compaction requested=false 2024-11-11T22:27:27,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T22:27:27,993 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:27,993 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-11T22:27:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T22:27:27,997 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:27,998 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:27,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T22:27:28,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T22:27:28,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:28,150 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:27:28,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/bffb1f7baf1f4d6283f16bbf261d6098 is 50, key is test_row_0/A:col10/1731364047555/Put/seqid=0 2024-11-11T22:27:28,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742283_1459 (size=12151) 2024-11-11T22:27:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:28,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:28,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364108244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:28,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364108254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T22:27:28,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364108363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:28,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364108363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:28,560 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/bffb1f7baf1f4d6283f16bbf261d6098 2024-11-11T22:27:28,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/935366e9a59d41629196f8662ceac2d6 is 50, key is test_row_0/B:col10/1731364047555/Put/seqid=0 2024-11-11T22:27:28,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742284_1460 (size=12151) 2024-11-11T22:27:28,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364108573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:28,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:28,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364108573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:28,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T22:27:28,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364108892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364108893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:28,969 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/935366e9a59d41629196f8662ceac2d6 2024-11-11T22:27:28,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/60795e6ef4d6481ca026e0f00350a9ce is 50, key is test_row_0/C:col10/1731364047555/Put/seqid=0 2024-11-11T22:27:28,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742285_1461 (size=12151) 2024-11-11T22:27:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T22:27:29,382 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/60795e6ef4d6481ca026e0f00350a9ce 2024-11-11T22:27:29,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/bffb1f7baf1f4d6283f16bbf261d6098 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/bffb1f7baf1f4d6283f16bbf261d6098 2024-11-11T22:27:29,387 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/bffb1f7baf1f4d6283f16bbf261d6098, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T22:27:29,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/935366e9a59d41629196f8662ceac2d6 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/935366e9a59d41629196f8662ceac2d6 2024-11-11T22:27:29,390 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/935366e9a59d41629196f8662ceac2d6, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T22:27:29,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/60795e6ef4d6481ca026e0f00350a9ce as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/60795e6ef4d6481ca026e0f00350a9ce 2024-11-11T22:27:29,395 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/60795e6ef4d6481ca026e0f00350a9ce, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T22:27:29,395 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 324e6f269deb5f32d638f5ef06ba3bab in 1245ms, sequenceid=153, compaction requested=true 2024-11-11T22:27:29,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:29,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:29,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-11T22:27:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-11T22:27:29,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T22:27:29,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3990 sec 2024-11-11T22:27:29,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4050 sec 2024-11-11T22:27:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:29,414 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:29,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T22:27:29,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/645d8ca516cc4317a4e7de91db5ccff4 is 50, key is test_row_0/A:col10/1731364049409/Put/seqid=0 2024-11-11T22:27:29,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742286_1462 (size=14541) 2024-11-11T22:27:29,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/645d8ca516cc4317a4e7de91db5ccff4 2024-11-11T22:27:29,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/309f686eb844480f8021dd5c703848c7 is 50, key is test_row_0/B:col10/1731364049409/Put/seqid=0 2024-11-11T22:27:29,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742287_1463 (size=12151) 2024-11-11T22:27:29,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364109493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:29,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364109497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:29,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364109603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:29,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364109612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:29,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364109813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:29,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364109823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:29,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/309f686eb844480f8021dd5c703848c7 2024-11-11T22:27:29,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/beeea6a47ccb418ba488a1b482e06e5d is 50, key is test_row_0/C:col10/1731364049409/Put/seqid=0 2024-11-11T22:27:29,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742288_1464 (size=12151) 2024-11-11T22:27:30,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364110133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T22:27:30,143 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:30,144 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-11T22:27:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:27:30,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:30,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:30,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:30,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364110143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:27:30,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/beeea6a47ccb418ba488a1b482e06e5d 2024-11-11T22:27:30,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/645d8ca516cc4317a4e7de91db5ccff4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/645d8ca516cc4317a4e7de91db5ccff4 2024-11-11T22:27:30,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/645d8ca516cc4317a4e7de91db5ccff4, entries=200, sequenceid=166, filesize=14.2 K 2024-11-11T22:27:30,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/309f686eb844480f8021dd5c703848c7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/309f686eb844480f8021dd5c703848c7 2024-11-11T22:27:30,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/309f686eb844480f8021dd5c703848c7, entries=150, sequenceid=166, filesize=11.9 K 2024-11-11T22:27:30,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/beeea6a47ccb418ba488a1b482e06e5d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/beeea6a47ccb418ba488a1b482e06e5d 2024-11-11T22:27:30,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/beeea6a47ccb418ba488a1b482e06e5d, entries=150, sequenceid=166, filesize=11.9 K 2024-11-11T22:27:30,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 324e6f269deb5f32d638f5ef06ba3bab in 858ms, sequenceid=166, compaction requested=true 2024-11-11T22:27:30,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:30,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:30,272 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:30,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:30,272 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:30,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:30,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:30,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:30,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:30,274 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53274 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:30,274 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/A is initiating minor compaction (all files) 2024-11-11T22:27:30,274 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/A in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:30,274 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/d44353d4adc949069bc4736095c390a0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/adc59025ec6b4572ae3427fb3993b3be, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/bffb1f7baf1f4d6283f16bbf261d6098, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/645d8ca516cc4317a4e7de91db5ccff4] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=52.0 K 2024-11-11T22:27:30,274 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:30,274 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting d44353d4adc949069bc4736095c390a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731364045949 2024-11-11T22:27:30,274 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/B is initiating minor compaction (all files) 2024-11-11T22:27:30,274 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/B in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:30,274 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting adc59025ec6b4572ae3427fb3993b3be, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731364046304 2024-11-11T22:27:30,274 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/45378cf341b6446598c5eda1c897a957, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3fa60c18f2524a27a8608f6cfe622b1e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/935366e9a59d41629196f8662ceac2d6, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/309f686eb844480f8021dd5c703848c7] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=47.4 K 2024-11-11T22:27:30,275 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 45378cf341b6446598c5eda1c897a957, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731364045949 2024-11-11T22:27:30,275 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting bffb1f7baf1f4d6283f16bbf261d6098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731364047555 2024-11-11T22:27:30,275 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 645d8ca516cc4317a4e7de91db5ccff4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731364048230 2024-11-11T22:27:30,275 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fa60c18f2524a27a8608f6cfe622b1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731364046304 2024-11-11T22:27:30,275 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 935366e9a59d41629196f8662ceac2d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731364047555 2024-11-11T22:27:30,275 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 309f686eb844480f8021dd5c703848c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731364048239 2024-11-11T22:27:30,287 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:30,288 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#A#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:30,288 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/803ff70eff564701a329133c0f768fa0 is 50, key is test_row_0/B:col10/1731364049409/Put/seqid=0 2024-11-11T22:27:30,288 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/b97ccdd3e405484685c60484ac01bb25 is 50, key is test_row_0/A:col10/1731364049409/Put/seqid=0 2024-11-11T22:27:30,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742289_1465 (size=12527) 2024-11-11T22:27:30,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742290_1466 (size=12527) 2024-11-11T22:27:30,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:27:30,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:30,309 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:27:30,313 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/b97ccdd3e405484685c60484ac01bb25 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/b97ccdd3e405484685c60484ac01bb25 2024-11-11T22:27:30,313 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/803ff70eff564701a329133c0f768fa0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/803ff70eff564701a329133c0f768fa0 2024-11-11T22:27:30,317 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/A of 324e6f269deb5f32d638f5ef06ba3bab into b97ccdd3e405484685c60484ac01bb25(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:30,317 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:30,317 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/A, priority=12, startTime=1731364050272; duration=0sec 2024-11-11T22:27:30,317 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:30,318 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:A 2024-11-11T22:27:30,318 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:27:30,318 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/B of 324e6f269deb5f32d638f5ef06ba3bab into 803ff70eff564701a329133c0f768fa0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:30,318 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:30,318 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/B, priority=12, startTime=1731364050272; duration=0sec 2024-11-11T22:27:30,318 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:30,318 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:B 2024-11-11T22:27:30,318 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:27:30,318 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/C is initiating minor compaction (all files) 2024-11-11T22:27:30,319 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/C in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:30,319 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dd169a2958e046f68e352494782b8d0b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/08410c00ea254ab4b86be4fadc0abcf6, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/60795e6ef4d6481ca026e0f00350a9ce, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/beeea6a47ccb418ba488a1b482e06e5d] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=47.4 K 2024-11-11T22:27:30,319 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd169a2958e046f68e352494782b8d0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731364045949 2024-11-11T22:27:30,319 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08410c00ea254ab4b86be4fadc0abcf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731364046304 2024-11-11T22:27:30,319 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60795e6ef4d6481ca026e0f00350a9ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731364047555 2024-11-11T22:27:30,319 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting beeea6a47ccb418ba488a1b482e06e5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731364048239 2024-11-11T22:27:30,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/f9bd12d9b06f443984cad5e62574f0f2 is 50, key is test_row_0/A:col10/1731364049495/Put/seqid=0 2024-11-11T22:27:30,330 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:30,331 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/516cd2553da344ecaa78cc023ecaa85a is 50, key is test_row_0/C:col10/1731364049409/Put/seqid=0 2024-11-11T22:27:30,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742291_1467 (size=12151) 2024-11-11T22:27:30,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742292_1468 (size=12527) 2024-11-11T22:27:30,359 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/516cd2553da344ecaa78cc023ecaa85a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/516cd2553da344ecaa78cc023ecaa85a 2024-11-11T22:27:30,367 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/C of 324e6f269deb5f32d638f5ef06ba3bab into 516cd2553da344ecaa78cc023ecaa85a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:30,367 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:30,367 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/C, priority=12, startTime=1731364050272; duration=0sec 2024-11-11T22:27:30,367 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:30,367 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:C 2024-11-11T22:27:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:27:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:30,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:30,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364110676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:30,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364110677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:30,734 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/f9bd12d9b06f443984cad5e62574f0f2 2024-11-11T22:27:30,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6ba468026fab4630beb8e5d706f055c1 is 50, key is test_row_0/B:col10/1731364049495/Put/seqid=0 2024-11-11T22:27:30,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742293_1469 (size=12151) 2024-11-11T22:27:30,755 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6ba468026fab4630beb8e5d706f055c1 2024-11-11T22:27:30,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/e2f1c0923f474536b9fe7328f8c18c9d is 50, key is test_row_0/C:col10/1731364049495/Put/seqid=0 2024-11-11T22:27:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:27:30,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742294_1470 (size=12151) 2024-11-11T22:27:30,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364110793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:30,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:30,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364110793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:30,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:30,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364111003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364111003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,178 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/e2f1c0923f474536b9fe7328f8c18c9d 2024-11-11T22:27:31,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/f9bd12d9b06f443984cad5e62574f0f2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/f9bd12d9b06f443984cad5e62574f0f2 2024-11-11T22:27:31,189 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/f9bd12d9b06f443984cad5e62574f0f2, entries=150, sequenceid=189, filesize=11.9 K 2024-11-11T22:27:31,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6ba468026fab4630beb8e5d706f055c1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6ba468026fab4630beb8e5d706f055c1 2024-11-11T22:27:31,193 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6ba468026fab4630beb8e5d706f055c1, entries=150, sequenceid=189, filesize=11.9 K 2024-11-11T22:27:31,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/e2f1c0923f474536b9fe7328f8c18c9d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/e2f1c0923f474536b9fe7328f8c18c9d 2024-11-11T22:27:31,197 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/e2f1c0923f474536b9fe7328f8c18c9d, entries=150, sequenceid=189, filesize=11.9 K 2024-11-11T22:27:31,198 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 324e6f269deb5f32d638f5ef06ba3bab in 889ms, sequenceid=189, compaction requested=false 2024-11-11T22:27:31,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:31,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:31,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-11T22:27:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-11T22:27:31,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T22:27:31,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0540 sec 2024-11-11T22:27:31,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.0650 sec 2024-11-11T22:27:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:27:31,283 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:31,283 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-11T22:27:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:27:31,286 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:31,288 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:31,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:31,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:27:31,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/3958081260c8407f83ea69a04d9663de is 50, key is test_row_0/A:col10/1731364050673/Put/seqid=0 2024-11-11T22:27:31,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742295_1471 (size=16931) 2024-11-11T22:27:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:27:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364111399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364111407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T22:27:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:31,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364111513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364111523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T22:27:31,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:31,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:31,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:31,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:27:31,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364111603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,605 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364111626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364111626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,633 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,634 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364111723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364111742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:31,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/3958081260c8407f83ea69a04d9663de 2024-11-11T22:27:31,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/ec40ed7c239d4509be7898cdedac0f5b is 50, key is test_row_0/B:col10/1731364050673/Put/seqid=0 2024-11-11T22:27:31,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742296_1472 (size=12151) 2024-11-11T22:27:31,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T22:27:31,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:31,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:31,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:31,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:27:31,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T22:27:31,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:31,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:31,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:31,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364112043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:32,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364112053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:32,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T22:27:32,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:32,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/ec40ed7c239d4509be7898cdedac0f5b 2024-11-11T22:27:32,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/dfc0660bd1794dd4a490223188544bab is 50, key is test_row_0/C:col10/1731364050673/Put/seqid=0 2024-11-11T22:27:32,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742297_1473 (size=12151) 2024-11-11T22:27:32,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T22:27:32,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:32,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T22:27:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:27:32,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T22:27:32,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:32,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:32,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364112563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:32,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:32,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364112573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:32,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/dfc0660bd1794dd4a490223188544bab 2024-11-11T22:27:32,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/3958081260c8407f83ea69a04d9663de as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/3958081260c8407f83ea69a04d9663de 2024-11-11T22:27:32,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/3958081260c8407f83ea69a04d9663de, entries=250, sequenceid=207, filesize=16.5 K 2024-11-11T22:27:32,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/ec40ed7c239d4509be7898cdedac0f5b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ec40ed7c239d4509be7898cdedac0f5b 2024-11-11T22:27:32,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ec40ed7c239d4509be7898cdedac0f5b, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T22:27:32,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/dfc0660bd1794dd4a490223188544bab as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dfc0660bd1794dd4a490223188544bab 2024-11-11T22:27:32,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dfc0660bd1794dd4a490223188544bab, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T22:27:32,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 324e6f269deb5f32d638f5ef06ba3bab in 1287ms, sequenceid=207, compaction requested=true 2024-11-11T22:27:32,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:32,617 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:32,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:32,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:32,617 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:32,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:32,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:32,618 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:32,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:32,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:32,618 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/A is initiating minor compaction (all files) 2024-11-11T22:27:32,618 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/A in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,618 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/b97ccdd3e405484685c60484ac01bb25, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/f9bd12d9b06f443984cad5e62574f0f2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/3958081260c8407f83ea69a04d9663de] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=40.6 K 2024-11-11T22:27:32,620 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting b97ccdd3e405484685c60484ac01bb25, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731364048239 2024-11-11T22:27:32,621 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f9bd12d9b06f443984cad5e62574f0f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731364049487 2024-11-11T22:27:32,621 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:32,621 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/B is initiating minor compaction (all files) 2024-11-11T22:27:32,621 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/B in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,622 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/803ff70eff564701a329133c0f768fa0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6ba468026fab4630beb8e5d706f055c1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ec40ed7c239d4509be7898cdedac0f5b] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=36.0 K 2024-11-11T22:27:32,622 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 3958081260c8407f83ea69a04d9663de, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731364050672 2024-11-11T22:27:32,622 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 803ff70eff564701a329133c0f768fa0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731364048239 2024-11-11T22:27:32,622 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba468026fab4630beb8e5d706f055c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731364049487 2024-11-11T22:27:32,623 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec40ed7c239d4509be7898cdedac0f5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731364050673 2024-11-11T22:27:32,635 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:32,635 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/540dc3942bd14fbfb21aab596c3d4321 is 50, key is test_row_0/B:col10/1731364050673/Put/seqid=0 2024-11-11T22:27:32,638 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#A#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:32,638 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/98d1b161fc4240f9982f69649af670bb is 50, key is test_row_0/A:col10/1731364050673/Put/seqid=0 2024-11-11T22:27:32,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742298_1474 (size=12629) 2024-11-11T22:27:32,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742299_1475 (size=12629) 2024-11-11T22:27:32,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T22:27:32,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:32,719 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T22:27:32,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/a097ba7f52d547f28e3e3e9d50a671fd is 50, key is test_row_0/A:col10/1731364051395/Put/seqid=0 2024-11-11T22:27:32,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742300_1476 (size=12151) 2024-11-11T22:27:33,058 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/540dc3942bd14fbfb21aab596c3d4321 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/540dc3942bd14fbfb21aab596c3d4321 2024-11-11T22:27:33,065 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/B of 324e6f269deb5f32d638f5ef06ba3bab into 540dc3942bd14fbfb21aab596c3d4321(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:33,065 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:33,065 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/B, priority=13, startTime=1731364052617; duration=0sec 2024-11-11T22:27:33,065 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:33,065 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:B 2024-11-11T22:27:33,065 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:33,067 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/98d1b161fc4240f9982f69649af670bb as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/98d1b161fc4240f9982f69649af670bb 2024-11-11T22:27:33,068 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:33,068 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/C is initiating minor compaction (all files) 2024-11-11T22:27:33,068 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/C in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:33,069 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/516cd2553da344ecaa78cc023ecaa85a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/e2f1c0923f474536b9fe7328f8c18c9d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dfc0660bd1794dd4a490223188544bab] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=36.0 K 2024-11-11T22:27:33,069 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 516cd2553da344ecaa78cc023ecaa85a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731364048239 2024-11-11T22:27:33,069 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2f1c0923f474536b9fe7328f8c18c9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731364049487 2024-11-11T22:27:33,070 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfc0660bd1794dd4a490223188544bab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731364050673 2024-11-11T22:27:33,072 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/A of 324e6f269deb5f32d638f5ef06ba3bab into 98d1b161fc4240f9982f69649af670bb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:33,072 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:33,072 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/A, priority=13, startTime=1731364052617; duration=0sec 2024-11-11T22:27:33,072 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:33,072 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:A 2024-11-11T22:27:33,076 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#C#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:33,077 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/c244971d8bdc436c9deedcf81ab1eedd is 50, key is test_row_0/C:col10/1731364050673/Put/seqid=0 2024-11-11T22:27:33,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742301_1477 (size=12629) 2024-11-11T22:27:33,083 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/c244971d8bdc436c9deedcf81ab1eedd as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c244971d8bdc436c9deedcf81ab1eedd 2024-11-11T22:27:33,086 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/C of 324e6f269deb5f32d638f5ef06ba3bab into c244971d8bdc436c9deedcf81ab1eedd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:33,086 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:33,086 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/C, priority=13, startTime=1731364052618; duration=0sec 2024-11-11T22:27:33,086 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:33,086 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:C 2024-11-11T22:27:33,136 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/a097ba7f52d547f28e3e3e9d50a671fd 2024-11-11T22:27:33,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/57cf5fd488ac4979a68df080297b58a3 is 50, key is test_row_0/B:col10/1731364051395/Put/seqid=0 2024-11-11T22:27:33,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742302_1478 (size=12151) 2024-11-11T22:27:33,158 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/57cf5fd488ac4979a68df080297b58a3 2024-11-11T22:27:33,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/a50ca3df911d44869713b0f9d5892e5d is 50, key is test_row_0/C:col10/1731364051395/Put/seqid=0 2024-11-11T22:27:33,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742303_1479 (size=12151) 2024-11-11T22:27:33,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:27:33,570 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/a50ca3df911d44869713b0f9d5892e5d 2024-11-11T22:27:33,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/a097ba7f52d547f28e3e3e9d50a671fd as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a097ba7f52d547f28e3e3e9d50a671fd 2024-11-11T22:27:33,577 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a097ba7f52d547f28e3e3e9d50a671fd, entries=150, sequenceid=228, filesize=11.9 K 2024-11-11T22:27:33,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/57cf5fd488ac4979a68df080297b58a3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/57cf5fd488ac4979a68df080297b58a3 2024-11-11T22:27:33,581 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/57cf5fd488ac4979a68df080297b58a3, entries=150, sequenceid=228, filesize=11.9 K 2024-11-11T22:27:33,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/a50ca3df911d44869713b0f9d5892e5d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a50ca3df911d44869713b0f9d5892e5d 2024-11-11T22:27:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:33,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:33,587 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a50ca3df911d44869713b0f9d5892e5d, entries=150, sequenceid=228, filesize=11.9 K 2024-11-11T22:27:33,588 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=13.42 KB/13740 for 324e6f269deb5f32d638f5ef06ba3bab in 869ms, sequenceid=228, compaction requested=false 2024-11-11T22:27:33,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:33,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:33,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T22:27:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-11T22:27:33,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T22:27:33,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3020 sec 2024-11-11T22:27:33,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.3080 sec 2024-11-11T22:27:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:33,605 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:33,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:27:33,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/344e745dcfd54e0b841c54f8e47fe953 is 50, key is test_row_0/A:col10/1731364053588/Put/seqid=0 2024-11-11T22:27:33,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742304_1480 (size=14541) 2024-11-11T22:27:33,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364113679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:33,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364113684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:33,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364113803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:33,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364113803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:34,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/344e745dcfd54e0b841c54f8e47fe953 2024-11-11T22:27:34,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:34,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364114014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364114014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:34,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/05d47171c4ff4c4b90e69fcd4bb7a11e is 50, key is test_row_0/B:col10/1731364053588/Put/seqid=0 2024-11-11T22:27:34,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742305_1481 (size=12151) 2024-11-11T22:27:34,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364114333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364114333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:34,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/05d47171c4ff4c4b90e69fcd4bb7a11e 2024-11-11T22:27:34,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/4d7d3c40ef0043d4b72c5b8128039102 is 50, key is test_row_0/C:col10/1731364053588/Put/seqid=0 2024-11-11T22:27:34,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742306_1482 (size=12151) 2024-11-11T22:27:34,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/4d7d3c40ef0043d4b72c5b8128039102 2024-11-11T22:27:34,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/344e745dcfd54e0b841c54f8e47fe953 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/344e745dcfd54e0b841c54f8e47fe953 2024-11-11T22:27:34,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364114843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:34,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:34,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/344e745dcfd54e0b841c54f8e47fe953, entries=200, sequenceid=242, filesize=14.2 K 2024-11-11T22:27:34,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/05d47171c4ff4c4b90e69fcd4bb7a11e as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/05d47171c4ff4c4b90e69fcd4bb7a11e 2024-11-11T22:27:34,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364114853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:34,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/05d47171c4ff4c4b90e69fcd4bb7a11e, entries=150, sequenceid=242, filesize=11.9 K 2024-11-11T22:27:34,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/4d7d3c40ef0043d4b72c5b8128039102 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/4d7d3c40ef0043d4b72c5b8128039102 2024-11-11T22:27:34,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/4d7d3c40ef0043d4b72c5b8128039102, entries=150, sequenceid=242, filesize=11.9 K 2024-11-11T22:27:34,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 324e6f269deb5f32d638f5ef06ba3bab in 1263ms, sequenceid=242, compaction requested=true 2024-11-11T22:27:34,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:34,869 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:27:34,869 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:34,869 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:34,869 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/A is initiating minor compaction (all files) 2024-11-11T22:27:34,870 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/A in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:34,870 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/98d1b161fc4240f9982f69649af670bb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a097ba7f52d547f28e3e3e9d50a671fd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/344e745dcfd54e0b841c54f8e47fe953] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=38.4 K 2024-11-11T22:27:34,870 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98d1b161fc4240f9982f69649af670bb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731364050673 2024-11-11T22:27:34,870 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting a097ba7f52d547f28e3e3e9d50a671fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731364051388 2024-11-11T22:27:34,871 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:34,871 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 344e745dcfd54e0b841c54f8e47fe953, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731364053588 2024-11-11T22:27:34,871 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/B is initiating minor compaction (all files) 2024-11-11T22:27:34,871 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/B in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:34,871 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/540dc3942bd14fbfb21aab596c3d4321, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/57cf5fd488ac4979a68df080297b58a3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/05d47171c4ff4c4b90e69fcd4bb7a11e] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=36.1 K 2024-11-11T22:27:34,871 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 540dc3942bd14fbfb21aab596c3d4321, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731364050673 2024-11-11T22:27:34,871 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 57cf5fd488ac4979a68df080297b58a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731364051388 2024-11-11T22:27:34,871 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 05d47171c4ff4c4b90e69fcd4bb7a11e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731364053588 2024-11-11T22:27:34,878 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:34,878 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6bf9fbfcd06e4759bacfabfec6bbb45c is 50, key is test_row_0/B:col10/1731364053588/Put/seqid=0 2024-11-11T22:27:34,880 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:34,880 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/384d54048d42446ca125270626077866 is 50, key is test_row_0/A:col10/1731364053588/Put/seqid=0 2024-11-11T22:27:34,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742307_1483 (size=12731) 2024-11-11T22:27:34,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742308_1484 (size=12731) 2024-11-11T22:27:35,295 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/384d54048d42446ca125270626077866 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/384d54048d42446ca125270626077866 2024-11-11T22:27:35,296 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6bf9fbfcd06e4759bacfabfec6bbb45c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bf9fbfcd06e4759bacfabfec6bbb45c 2024-11-11T22:27:35,300 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/A of 324e6f269deb5f32d638f5ef06ba3bab into 384d54048d42446ca125270626077866(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:35,300 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:35,300 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/A, priority=13, startTime=1731364054868; duration=0sec 2024-11-11T22:27:35,300 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:35,300 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:A 2024-11-11T22:27:35,300 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:35,300 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/B of 324e6f269deb5f32d638f5ef06ba3bab into 6bf9fbfcd06e4759bacfabfec6bbb45c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:35,300 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:35,300 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/B, priority=13, startTime=1731364054869; duration=0sec 2024-11-11T22:27:35,301 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:35,301 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:B 2024-11-11T22:27:35,301 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:35,301 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/C is initiating minor compaction (all files) 2024-11-11T22:27:35,301 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/C in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:35,301 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c244971d8bdc436c9deedcf81ab1eedd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a50ca3df911d44869713b0f9d5892e5d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/4d7d3c40ef0043d4b72c5b8128039102] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=36.1 K 2024-11-11T22:27:35,302 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting c244971d8bdc436c9deedcf81ab1eedd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731364050673 2024-11-11T22:27:35,302 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting a50ca3df911d44869713b0f9d5892e5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731364051388 2024-11-11T22:27:35,302 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d7d3c40ef0043d4b72c5b8128039102, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731364053588 2024-11-11T22:27:35,307 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:35,308 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/695e5fe2e22b4a918baafb9825780915 is 50, key is test_row_0/C:col10/1731364053588/Put/seqid=0 2024-11-11T22:27:35,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742309_1485 (size=12731) 2024-11-11T22:27:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:27:35,443 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:35,444 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-11T22:27:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T22:27:35,445 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:35,446 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:35,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T22:27:35,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T22:27:35,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:35,598 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T22:27:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/8e800f73b7524f85aa66f3fd5f5e87b9 is 50, key is test_row_0/A:col10/1731364053678/Put/seqid=0 2024-11-11T22:27:35,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742310_1486 (size=12301) 2024-11-11T22:27:35,727 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/695e5fe2e22b4a918baafb9825780915 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/695e5fe2e22b4a918baafb9825780915 2024-11-11T22:27:35,731 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/C of 324e6f269deb5f32d638f5ef06ba3bab into 695e5fe2e22b4a918baafb9825780915(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:35,731 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:35,731 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/C, priority=13, startTime=1731364054869; duration=0sec 2024-11-11T22:27:35,731 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:35,731 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:C 2024-11-11T22:27:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T22:27:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:35,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:35,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364115891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:35,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364115893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364116003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,012 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/8e800f73b7524f85aa66f3fd5f5e87b9 2024-11-11T22:27:36,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/a53e0f7e342347e8b8d20d663cb40ba0 is 50, key is test_row_0/B:col10/1731364053678/Put/seqid=0 2024-11-11T22:27:36,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:36,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742311_1487 (size=12301) 2024-11-11T22:27:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364116014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:36,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T22:27:36,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364116212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364116233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,422 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/a53e0f7e342347e8b8d20d663cb40ba0 2024-11-11T22:27:36,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/29f8ba28491e418db93dfe4462e04ee2 is 50, key is test_row_0/C:col10/1731364053678/Put/seqid=0 2024-11-11T22:27:36,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742312_1488 (size=12301) 2024-11-11T22:27:36,442 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/29f8ba28491e418db93dfe4462e04ee2 2024-11-11T22:27:36,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/8e800f73b7524f85aa66f3fd5f5e87b9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/8e800f73b7524f85aa66f3fd5f5e87b9 2024-11-11T22:27:36,449 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/8e800f73b7524f85aa66f3fd5f5e87b9, entries=150, sequenceid=269, filesize=12.0 K 2024-11-11T22:27:36,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/a53e0f7e342347e8b8d20d663cb40ba0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/a53e0f7e342347e8b8d20d663cb40ba0 2024-11-11T22:27:36,459 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/a53e0f7e342347e8b8d20d663cb40ba0, entries=150, sequenceid=269, filesize=12.0 K 2024-11-11T22:27:36,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/29f8ba28491e418db93dfe4462e04ee2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/29f8ba28491e418db93dfe4462e04ee2 2024-11-11T22:27:36,465 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/29f8ba28491e418db93dfe4462e04ee2, entries=150, sequenceid=269, filesize=12.0 K 2024-11-11T22:27:36,466 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 324e6f269deb5f32d638f5ef06ba3bab in 868ms, sequenceid=269, compaction requested=false 2024-11-11T22:27:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T22:27:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-11T22:27:36,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T22:27:36,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0210 sec 2024-11-11T22:27:36,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0260 sec 2024-11-11T22:27:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:36,527 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:36,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T22:27:36,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/2e0e0deb8f1f466896beef1eb379fbc4 is 50, key is test_row_0/A:col10/1731364055884/Put/seqid=0 2024-11-11T22:27:36,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742313_1489 (size=14741) 2024-11-11T22:27:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T22:27:36,583 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:36,583 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:36,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364116605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364116611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-11T22:27:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:27:36,639 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:36,640 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:36,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:36,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364116723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364116723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:27:36,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T22:27:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:36,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:36,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:36,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/2e0e0deb8f1f466896beef1eb379fbc4 2024-11-11T22:27:36,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/3138a34676dc4ebf8c49456ca61dd438 is 50, key is test_row_0/B:col10/1731364055884/Put/seqid=0 2024-11-11T22:27:36,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364116943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:36,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:36,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364116943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:36,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:36,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:36,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T22:27:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:36,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:27:36,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742314_1490 (size=12301) 2024-11-11T22:27:37,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T22:27:37,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:37,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:37,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T22:27:37,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:37,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:37,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:37,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:37,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:37,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364117253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364117253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:27:37,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/3138a34676dc4ebf8c49456ca61dd438 2024-11-11T22:27:37,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/30630256bdec437bb6e78d23b784c0c8 is 50, key is test_row_0/C:col10/1731364055884/Put/seqid=0 2024-11-11T22:27:37,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742315_1491 (size=12301) 2024-11-11T22:27:37,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/30630256bdec437bb6e78d23b784c0c8 2024-11-11T22:27:37,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/2e0e0deb8f1f466896beef1eb379fbc4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2e0e0deb8f1f466896beef1eb379fbc4 2024-11-11T22:27:37,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2e0e0deb8f1f466896beef1eb379fbc4, entries=200, sequenceid=282, filesize=14.4 K 2024-11-11T22:27:37,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/3138a34676dc4ebf8c49456ca61dd438 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3138a34676dc4ebf8c49456ca61dd438 2024-11-11T22:27:37,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3138a34676dc4ebf8c49456ca61dd438, entries=150, sequenceid=282, filesize=12.0 K 2024-11-11T22:27:37,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/30630256bdec437bb6e78d23b784c0c8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/30630256bdec437bb6e78d23b784c0c8 2024-11-11T22:27:37,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/30630256bdec437bb6e78d23b784c0c8, entries=150, sequenceid=282, filesize=12.0 K 2024-11-11T22:27:37,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 324e6f269deb5f32d638f5ef06ba3bab in 870ms, sequenceid=282, compaction requested=true 2024-11-11T22:27:37,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:37,397 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:37,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:37,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:37,397 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:37,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:37,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:37,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:37,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:37,399 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:37,399 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/B is initiating minor compaction (all files) 2024-11-11T22:27:37,399 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/B in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:37,399 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bf9fbfcd06e4759bacfabfec6bbb45c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/a53e0f7e342347e8b8d20d663cb40ba0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3138a34676dc4ebf8c49456ca61dd438] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=36.5 K 2024-11-11T22:27:37,399 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:37,399 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/A is initiating minor compaction (all files) 2024-11-11T22:27:37,399 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/A in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:37,399 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/384d54048d42446ca125270626077866, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/8e800f73b7524f85aa66f3fd5f5e87b9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2e0e0deb8f1f466896beef1eb379fbc4] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=38.8 K 2024-11-11T22:27:37,400 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf9fbfcd06e4759bacfabfec6bbb45c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731364053588 2024-11-11T22:27:37,400 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384d54048d42446ca125270626077866, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731364053588 2024-11-11T22:27:37,400 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a53e0f7e342347e8b8d20d663cb40ba0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731364053676 2024-11-11T22:27:37,400 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e800f73b7524f85aa66f3fd5f5e87b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731364053676 2024-11-11T22:27:37,400 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 3138a34676dc4ebf8c49456ca61dd438, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731364055875 2024-11-11T22:27:37,400 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e0e0deb8f1f466896beef1eb379fbc4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731364055875 2024-11-11T22:27:37,407 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:37,408 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/b49c64a4c1fd4af7b82c226086c91d9f is 50, key is test_row_0/B:col10/1731364055884/Put/seqid=0 2024-11-11T22:27:37,408 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#A#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:37,409 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/b539bc48a19546fea8c8ae2f3ea48c2b is 50, key is test_row_0/A:col10/1731364055884/Put/seqid=0 2024-11-11T22:27:37,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T22:27:37,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:37,411 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:27:37,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742317_1493 (size=12983) 2024-11-11T22:27:37,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742316_1492 (size=12983) 2024-11-11T22:27:37,431 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/b49c64a4c1fd4af7b82c226086c91d9f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/b49c64a4c1fd4af7b82c226086c91d9f 2024-11-11T22:27:37,436 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/B of 324e6f269deb5f32d638f5ef06ba3bab into b49c64a4c1fd4af7b82c226086c91d9f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:37,436 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:37,436 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/B, priority=13, startTime=1731364057397; duration=0sec 2024-11-11T22:27:37,436 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:37,436 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:B 2024-11-11T22:27:37,436 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/11831260519945c597b4071bdcaba71a is 50, key is test_row_0/A:col10/1731364056610/Put/seqid=0 2024-11-11T22:27:37,445 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:37,445 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/C is initiating minor compaction (all files) 2024-11-11T22:27:37,445 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/C in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:37,445 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/695e5fe2e22b4a918baafb9825780915, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/29f8ba28491e418db93dfe4462e04ee2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/30630256bdec437bb6e78d23b784c0c8] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=36.5 K 2024-11-11T22:27:37,446 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 695e5fe2e22b4a918baafb9825780915, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731364053588 2024-11-11T22:27:37,446 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 29f8ba28491e418db93dfe4462e04ee2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731364053676 2024-11-11T22:27:37,446 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 30630256bdec437bb6e78d23b784c0c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731364055875 2024-11-11T22:27:37,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742318_1494 (size=12301) 2024-11-11T22:27:37,455 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#C#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:37,455 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/99aab58caf574724942c220fde93c7d7 is 50, key is test_row_0/C:col10/1731364055884/Put/seqid=0 2024-11-11T22:27:37,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742319_1495 (size=12983) 2024-11-11T22:27:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:27:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:37,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:37,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364117803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:37,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364117804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:37,832 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/b539bc48a19546fea8c8ae2f3ea48c2b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/b539bc48a19546fea8c8ae2f3ea48c2b 2024-11-11T22:27:37,836 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/A of 324e6f269deb5f32d638f5ef06ba3bab into b539bc48a19546fea8c8ae2f3ea48c2b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:37,836 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:37,836 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/A, priority=13, startTime=1731364057397; duration=0sec 2024-11-11T22:27:37,836 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:37,836 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:A 2024-11-11T22:27:37,847 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/11831260519945c597b4071bdcaba71a 2024-11-11T22:27:37,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/fa7c6f6cf988408a954a042a009b9bbf is 50, key is test_row_0/B:col10/1731364056610/Put/seqid=0 2024-11-11T22:27:37,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742320_1496 (size=12301) 2024-11-11T22:27:37,874 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/99aab58caf574724942c220fde93c7d7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/99aab58caf574724942c220fde93c7d7 2024-11-11T22:27:37,878 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/C of 324e6f269deb5f32d638f5ef06ba3bab into 99aab58caf574724942c220fde93c7d7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:37,878 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:37,878 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/C, priority=13, startTime=1731364057397; duration=0sec 2024-11-11T22:27:37,878 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:37,878 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:C 2024-11-11T22:27:37,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364117913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:37,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364117923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:37,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:37,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:38,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364118123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:38,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:38,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364118133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:38,259 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/fa7c6f6cf988408a954a042a009b9bbf 2024-11-11T22:27:38,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/246ae43c51c54b2fbc5a06af30737ab8 is 50, key is test_row_0/C:col10/1731364056610/Put/seqid=0 2024-11-11T22:27:38,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742321_1497 (size=12301) 2024-11-11T22:27:38,279 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/246ae43c51c54b2fbc5a06af30737ab8 2024-11-11T22:27:38,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/11831260519945c597b4071bdcaba71a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/11831260519945c597b4071bdcaba71a 2024-11-11T22:27:38,287 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/11831260519945c597b4071bdcaba71a, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T22:27:38,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/fa7c6f6cf988408a954a042a009b9bbf as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/fa7c6f6cf988408a954a042a009b9bbf 2024-11-11T22:27:38,291 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/fa7c6f6cf988408a954a042a009b9bbf, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T22:27:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/246ae43c51c54b2fbc5a06af30737ab8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/246ae43c51c54b2fbc5a06af30737ab8 2024-11-11T22:27:38,302 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/246ae43c51c54b2fbc5a06af30737ab8, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T22:27:38,302 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 324e6f269deb5f32d638f5ef06ba3bab in 892ms, sequenceid=306, compaction requested=false 2024-11-11T22:27:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-11T22:27:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-11T22:27:38,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T22:27:38,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6640 sec 2024-11-11T22:27:38,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7340 sec 2024-11-11T22:27:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:38,435 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:38,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T22:27:38,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/a9adc40dc7e34c72aa82e216b56f2be8 is 50, key is test_row_0/A:col10/1731364057802/Put/seqid=0 2024-11-11T22:27:38,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742322_1498 (size=14741) 2024-11-11T22:27:38,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364118522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:38,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364118529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:38,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364118643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:38,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364118643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:27:38,783 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:38,783 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T22:27:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:27:38,785 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:38,785 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:38,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:38,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364118853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:38,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/a9adc40dc7e34c72aa82e216b56f2be8 2024-11-11T22:27:38,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364118853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:38,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/574891adba3e4600b65432dc1577fa0a is 50, key is test_row_0/B:col10/1731364057802/Put/seqid=0 2024-11-11T22:27:38,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742323_1499 (size=12301) 2024-11-11T22:27:38,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/574891adba3e4600b65432dc1577fa0a 2024-11-11T22:27:38,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/5d7e033aa5804060a7d93a1bf15fe919 is 50, key is test_row_0/C:col10/1731364057802/Put/seqid=0 2024-11-11T22:27:38,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742324_1500 (size=12301) 2024-11-11T22:27:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:27:38,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T22:27:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:38,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:39,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T22:27:39,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:39,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:39,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:39,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:39,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:27:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364119163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364119163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:39,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T22:27:39,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:39,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:39,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:39,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:39,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:39,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/5d7e033aa5804060a7d93a1bf15fe919 2024-11-11T22:27:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/a9adc40dc7e34c72aa82e216b56f2be8 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a9adc40dc7e34c72aa82e216b56f2be8 2024-11-11T22:27:39,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a9adc40dc7e34c72aa82e216b56f2be8, entries=200, sequenceid=322, filesize=14.4 K 2024-11-11T22:27:39,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/574891adba3e4600b65432dc1577fa0a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/574891adba3e4600b65432dc1577fa0a 2024-11-11T22:27:39,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/574891adba3e4600b65432dc1577fa0a, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T22:27:39,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/5d7e033aa5804060a7d93a1bf15fe919 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/5d7e033aa5804060a7d93a1bf15fe919 2024-11-11T22:27:39,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/5d7e033aa5804060a7d93a1bf15fe919, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T22:27:39,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 324e6f269deb5f32d638f5ef06ba3bab in 867ms, sequenceid=322, compaction requested=true 2024-11-11T22:27:39,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:39,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:39,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:39,302 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:39,302 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:39,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:39,303 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:39,303 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:39,303 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/A is initiating minor compaction (all files) 2024-11-11T22:27:39,303 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/B is initiating minor compaction (all files) 2024-11-11T22:27:39,303 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/A in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:39,303 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/B in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:39,303 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/b539bc48a19546fea8c8ae2f3ea48c2b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/11831260519945c597b4071bdcaba71a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a9adc40dc7e34c72aa82e216b56f2be8] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=39.1 K 2024-11-11T22:27:39,303 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/b49c64a4c1fd4af7b82c226086c91d9f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/fa7c6f6cf988408a954a042a009b9bbf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/574891adba3e4600b65432dc1577fa0a] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=36.7 K 2024-11-11T22:27:39,304 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting b49c64a4c1fd4af7b82c226086c91d9f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731364055875 2024-11-11T22:27:39,304 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b539bc48a19546fea8c8ae2f3ea48c2b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731364055875 2024-11-11T22:27:39,304 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11831260519945c597b4071bdcaba71a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731364056600 2024-11-11T22:27:39,304 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting fa7c6f6cf988408a954a042a009b9bbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731364056600 2024-11-11T22:27:39,304 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9adc40dc7e34c72aa82e216b56f2be8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731364057781 2024-11-11T22:27:39,304 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 574891adba3e4600b65432dc1577fa0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731364057781 2024-11-11T22:27:39,311 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:39,312 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/cb4c5aeb8c934fcf93230ec81ca94683 is 50, key is test_row_0/B:col10/1731364057802/Put/seqid=0 2024-11-11T22:27:39,314 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#A#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:39,314 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/c2ab73b5d462437eaf3a93ceb57484e0 is 50, key is test_row_0/A:col10/1731364057802/Put/seqid=0 2024-11-11T22:27:39,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742325_1501 (size=13085) 2024-11-11T22:27:39,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742326_1502 (size=13085) 2024-11-11T22:27:39,332 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/c2ab73b5d462437eaf3a93ceb57484e0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c2ab73b5d462437eaf3a93ceb57484e0 2024-11-11T22:27:39,333 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/cb4c5aeb8c934fcf93230ec81ca94683 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/cb4c5aeb8c934fcf93230ec81ca94683 2024-11-11T22:27:39,337 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/B of 324e6f269deb5f32d638f5ef06ba3bab into cb4c5aeb8c934fcf93230ec81ca94683(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:39,337 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:39,337 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/B, priority=13, startTime=1731364059302; duration=0sec 2024-11-11T22:27:39,337 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/A of 324e6f269deb5f32d638f5ef06ba3bab into c2ab73b5d462437eaf3a93ceb57484e0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:39,337 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:39,337 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:B 2024-11-11T22:27:39,337 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:39,337 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/A, priority=13, startTime=1731364059302; duration=0sec 2024-11-11T22:27:39,337 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:39,337 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:39,337 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:A 2024-11-11T22:27:39,338 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:39,338 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/C is initiating minor compaction (all files) 2024-11-11T22:27:39,338 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/C in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:39,338 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/99aab58caf574724942c220fde93c7d7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/246ae43c51c54b2fbc5a06af30737ab8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/5d7e033aa5804060a7d93a1bf15fe919] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=36.7 K 2024-11-11T22:27:39,338 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 99aab58caf574724942c220fde93c7d7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731364055875 2024-11-11T22:27:39,339 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 246ae43c51c54b2fbc5a06af30737ab8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731364056600 2024-11-11T22:27:39,339 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d7e033aa5804060a7d93a1bf15fe919, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731364057781 2024-11-11T22:27:39,344 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#C#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:39,345 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/2c8eb9128e1d448881ea196514b65100 is 50, key is test_row_0/C:col10/1731364057802/Put/seqid=0 2024-11-11T22:27:39,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742327_1503 (size=13085) 2024-11-11T22:27:39,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T22:27:39,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:39,398 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:27:39,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/7f8c1d14941b4aea8df4099a84f29ea7 is 50, key is test_row_0/A:col10/1731364058516/Put/seqid=0 2024-11-11T22:27:39,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742328_1504 (size=12301) 2024-11-11T22:27:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:27:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:39,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364119707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:39,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364119714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:39,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:39,751 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/2c8eb9128e1d448881ea196514b65100 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/2c8eb9128e1d448881ea196514b65100 2024-11-11T22:27:39,755 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/C of 324e6f269deb5f32d638f5ef06ba3bab into 2c8eb9128e1d448881ea196514b65100(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:39,755 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:39,755 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/C, priority=13, startTime=1731364059303; duration=0sec 2024-11-11T22:27:39,755 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:39,755 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:C 2024-11-11T22:27:39,806 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/7f8c1d14941b4aea8df4099a84f29ea7 2024-11-11T22:27:39,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/3f8455cb2bc24600aebcbc5f9b2c829a is 50, key is test_row_0/B:col10/1731364058516/Put/seqid=0 2024-11-11T22:27:39,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742329_1505 (size=12301) 2024-11-11T22:27:39,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364119823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:39,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364119833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:27:40,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364120033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:40,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364120042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:40,218 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/3f8455cb2bc24600aebcbc5f9b2c829a 2024-11-11T22:27:40,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/94d7261d98534907ab30767471a4a4fc is 50, key is test_row_0/C:col10/1731364058516/Put/seqid=0 2024-11-11T22:27:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742330_1506 (size=12301) 2024-11-11T22:27:40,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364120343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:40,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364120363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:40,637 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/94d7261d98534907ab30767471a4a4fc 2024-11-11T22:27:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/7f8c1d14941b4aea8df4099a84f29ea7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/7f8c1d14941b4aea8df4099a84f29ea7 2024-11-11T22:27:40,644 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/7f8c1d14941b4aea8df4099a84f29ea7, entries=150, sequenceid=347, filesize=12.0 K 2024-11-11T22:27:40,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/3f8455cb2bc24600aebcbc5f9b2c829a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3f8455cb2bc24600aebcbc5f9b2c829a 2024-11-11T22:27:40,648 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3f8455cb2bc24600aebcbc5f9b2c829a, entries=150, sequenceid=347, filesize=12.0 K 2024-11-11T22:27:40,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/94d7261d98534907ab30767471a4a4fc as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/94d7261d98534907ab30767471a4a4fc 2024-11-11T22:27:40,651 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/94d7261d98534907ab30767471a4a4fc, entries=150, sequenceid=347, filesize=12.0 K 2024-11-11T22:27:40,652 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 324e6f269deb5f32d638f5ef06ba3bab in 1254ms, sequenceid=347, compaction requested=false 2024-11-11T22:27:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T22:27:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T22:27:40,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T22:27:40,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-11-11T22:27:40,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8730 sec 2024-11-11T22:27:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:40,858 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:40,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:27:40,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/94d756717fe74abdad96f8b87b09a4fd is 50, key is test_row_0/A:col10/1731364060857/Put/seqid=0 2024-11-11T22:27:40,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742331_1507 (size=14741) 2024-11-11T22:27:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:27:40,943 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:40,944 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T22:27:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:27:40,945 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:40,946 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:40,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:40,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364120952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:40,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364120952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:27:41,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364121063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364121063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:41,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:27:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:27:41,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:41,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:27:41,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/94d756717fe74abdad96f8b87b09a4fd 2024-11-11T22:27:41,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364121273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:41,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364121273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:41,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/60ee4ca441884639bdfb84eb73b70d65 is 50, key is test_row_0/B:col10/1731364060857/Put/seqid=0 2024-11-11T22:27:41,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742332_1508 (size=12301) 2024-11-11T22:27:41,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:27:41,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:41,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:27:41,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:41,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:27:41,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364121583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364121583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:41,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38528 deadline: 1731364121613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:41,620 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_2' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:41,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38582 deadline: 1731364121653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:41,658 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:41,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/60ee4ca441884639bdfb84eb73b70d65 2024-11-11T22:27:41,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731364121683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:41,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/f36151e6222f484e97b0c1d9db5be997 is 50, key is test_row_0/C:col10/1731364060857/Put/seqid=0 2024-11-11T22:27:41,691 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,39613,1731363918313 for 'test_row_1' in 324e6f269deb5f32d638f5ef06ba3bab of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18316 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:27:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:41,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742333_1509 (size=12301) 2024-11-11T22:27:41,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:27:41,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:41,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:27:41,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:41,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:41,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:42,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:27:42,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:42,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:42,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:42,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:42,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:27:42,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/f36151e6222f484e97b0c1d9db5be997 2024-11-11T22:27:42,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/94d756717fe74abdad96f8b87b09a4fd as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/94d756717fe74abdad96f8b87b09a4fd 2024-11-11T22:27:42,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/94d756717fe74abdad96f8b87b09a4fd, entries=200, sequenceid=362, filesize=14.4 K 2024-11-11T22:27:42,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/60ee4ca441884639bdfb84eb73b70d65 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/60ee4ca441884639bdfb84eb73b70d65 2024-11-11T22:27:42,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38596 deadline: 1731364122103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:42,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38600 deadline: 1731364122103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 , the old value is region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., hostname=8d82b89dd27d,39613,1731363918313, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:27:42,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/60ee4ca441884639bdfb84eb73b70d65, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T22:27:42,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/f36151e6222f484e97b0c1d9db5be997 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f36151e6222f484e97b0c1d9db5be997 2024-11-11T22:27:42,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f36151e6222f484e97b0c1d9db5be997, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T22:27:42,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 324e6f269deb5f32d638f5ef06ba3bab in 1259ms, sequenceid=362, compaction requested=true 2024-11-11T22:27:42,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:42,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:27:42,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:42,117 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:42,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:27:42,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:42,117 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:42,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 324e6f269deb5f32d638f5ef06ba3bab:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:27:42,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:42,118 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:42,118 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:42,118 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/A is initiating minor compaction (all files) 2024-11-11T22:27:42,118 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/B is initiating minor compaction (all files) 2024-11-11T22:27:42,118 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/A in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:42,118 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/B in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:42,118 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c2ab73b5d462437eaf3a93ceb57484e0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/7f8c1d14941b4aea8df4099a84f29ea7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/94d756717fe74abdad96f8b87b09a4fd] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=39.2 K 2024-11-11T22:27:42,118 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/cb4c5aeb8c934fcf93230ec81ca94683, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3f8455cb2bc24600aebcbc5f9b2c829a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/60ee4ca441884639bdfb84eb73b70d65] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=36.8 K 2024-11-11T22:27:42,119 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting cb4c5aeb8c934fcf93230ec81ca94683, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731364057781 2024-11-11T22:27:42,119 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ab73b5d462437eaf3a93ceb57484e0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731364057781 2024-11-11T22:27:42,119 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f8c1d14941b4aea8df4099a84f29ea7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731364058490 2024-11-11T22:27:42,119 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f8455cb2bc24600aebcbc5f9b2c829a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731364058490 2024-11-11T22:27:42,119 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ee4ca441884639bdfb84eb73b70d65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731364059700 2024-11-11T22:27:42,120 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94d756717fe74abdad96f8b87b09a4fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731364059694 2024-11-11T22:27:42,135 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:42,135 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/254de0d378b84772aae2975884d62c62 is 50, key is test_row_0/B:col10/1731364060857/Put/seqid=0 2024-11-11T22:27:42,137 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#A#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:42,138 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/5706b6cb695945959fa020ff87b83f75 is 50, key is test_row_0/A:col10/1731364060857/Put/seqid=0 2024-11-11T22:27:42,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742334_1510 (size=13187) 2024-11-11T22:27:42,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742335_1511 (size=13187) 2024-11-11T22:27:42,155 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/5706b6cb695945959fa020ff87b83f75 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/5706b6cb695945959fa020ff87b83f75 2024-11-11T22:27:42,159 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/A of 324e6f269deb5f32d638f5ef06ba3bab into 5706b6cb695945959fa020ff87b83f75(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:42,160 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:42,160 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/A, priority=13, startTime=1731364062117; duration=0sec 2024-11-11T22:27:42,160 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:27:42,160 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:A 2024-11-11T22:27:42,160 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:27:42,160 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:27:42,160 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 324e6f269deb5f32d638f5ef06ba3bab/C is initiating minor compaction (all files) 2024-11-11T22:27:42,160 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 324e6f269deb5f32d638f5ef06ba3bab/C in TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:42,160 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/2c8eb9128e1d448881ea196514b65100, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/94d7261d98534907ab30767471a4a4fc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f36151e6222f484e97b0c1d9db5be997] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp, totalSize=36.8 K 2024-11-11T22:27:42,161 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c8eb9128e1d448881ea196514b65100, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731364057781 2024-11-11T22:27:42,161 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94d7261d98534907ab30767471a4a4fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731364058490 2024-11-11T22:27:42,161 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting f36151e6222f484e97b0c1d9db5be997, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731364059700 2024-11-11T22:27:42,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:27:42,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:42,169 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:27:42,169 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 324e6f269deb5f32d638f5ef06ba3bab#C#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:27:42,169 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/7e8b9127f4dd472c9b817be5f308f716 is 50, key is test_row_0/C:col10/1731364060857/Put/seqid=0 2024-11-11T22:27:42,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/9d92f07fa07d47539911b6ea3af1014a is 50, key is test_row_0/A:col10/1731364060950/Put/seqid=0 2024-11-11T22:27:42,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742337_1513 (size=12301) 2024-11-11T22:27:42,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742336_1512 (size=13187) 2024-11-11T22:27:42,546 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/254de0d378b84772aae2975884d62c62 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/254de0d378b84772aae2975884d62c62 2024-11-11T22:27:42,550 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/B of 324e6f269deb5f32d638f5ef06ba3bab into 254de0d378b84772aae2975884d62c62(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:42,550 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:42,550 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/B, priority=13, startTime=1731364062117; duration=0sec 2024-11-11T22:27:42,550 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:42,551 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:B 2024-11-11T22:27:42,590 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/9d92f07fa07d47539911b6ea3af1014a 2024-11-11T22:27:42,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/52f546625a994063997becd8e92501c9 is 50, key is test_row_0/B:col10/1731364060950/Put/seqid=0 2024-11-11T22:27:42,597 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/7e8b9127f4dd472c9b817be5f308f716 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/7e8b9127f4dd472c9b817be5f308f716 2024-11-11T22:27:42,609 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 324e6f269deb5f32d638f5ef06ba3bab/C of 324e6f269deb5f32d638f5ef06ba3bab into 7e8b9127f4dd472c9b817be5f308f716(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:27:42,609 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:42,609 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., storeName=324e6f269deb5f32d638f5ef06ba3bab/C, priority=13, startTime=1731364062117; duration=0sec 2024-11-11T22:27:42,610 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:27:42,610 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 324e6f269deb5f32d638f5ef06ba3bab:C 2024-11-11T22:27:42,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742338_1514 (size=12301) 2024-11-11T22:27:42,692 INFO [Thread-1960 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1960. 2024-11-11T22:27:42,692 DEBUG [Thread-1960 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:42,692 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:42,692 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:42,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:42,695 INFO [Thread-1954 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1954. 2024-11-11T22:27:42,695 DEBUG [Thread-1954 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:42,695 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:42,695 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:42,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:42,699 INFO [Thread-1952 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1952. 2024-11-11T22:27:42,699 DEBUG [Thread-1952 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:42,699 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:42,699 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:42,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:42,699 INFO [Thread-1958 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1958. 2024-11-11T22:27:42,699 DEBUG [Thread-1958 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:42,699 DEBUG [Thread-1958 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:42,699 DEBUG [Thread-1958 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:42,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:42,701 INFO [Thread-1956 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1956. 2024-11-11T22:27:42,701 DEBUG [Thread-1956 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:42,701 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:42,701 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:42,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:43,017 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/52f546625a994063997becd8e92501c9 2024-11-11T22:27:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/b9b0a353d4824f02a4e240ad4c67b33d is 50, key is test_row_0/C:col10/1731364060950/Put/seqid=0 2024-11-11T22:27:43,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742339_1515 (size=12301) 2024-11-11T22:27:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:27:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. as already flushing 2024-11-11T22:27:43,124 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-11-11T22:27:43,125 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:43,125 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:43,125 INFO [Thread-1947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1947. 2024-11-11T22:27:43,125 DEBUG [Thread-1947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:43,125 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:43,125 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:43,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:43,125 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:43,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:43,430 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/b9b0a353d4824f02a4e240ad4c67b33d 2024-11-11T22:27:43,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/9d92f07fa07d47539911b6ea3af1014a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/9d92f07fa07d47539911b6ea3af1014a 2024-11-11T22:27:43,435 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/9d92f07fa07d47539911b6ea3af1014a, entries=150, sequenceid=386, filesize=12.0 K 2024-11-11T22:27:43,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/52f546625a994063997becd8e92501c9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/52f546625a994063997becd8e92501c9 2024-11-11T22:27:43,439 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/52f546625a994063997becd8e92501c9, entries=150, sequenceid=386, filesize=12.0 K 2024-11-11T22:27:43,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/b9b0a353d4824f02a4e240ad4c67b33d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/b9b0a353d4824f02a4e240ad4c67b33d 2024-11-11T22:27:43,442 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/b9b0a353d4824f02a4e240ad4c67b33d, entries=150, sequenceid=386, filesize=12.0 K 2024-11-11T22:27:43,443 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=13.42 KB/13740 for 324e6f269deb5f32d638f5ef06ba3bab in 1275ms, sequenceid=386, compaction requested=false 2024-11-11T22:27:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 324e6f269deb5f32d638f5ef06ba3bab: 2024-11-11T22:27:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T22:27:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T22:27:43,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T22:27:43,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4980 sec 2024-11-11T22:27:43,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.5020 sec 2024-11-11T22:27:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:27:45,124 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:46,033 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:27:51,698 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-11-11T22:27:51,698 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:51,698 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:51,699 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:51,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:51,748 INFO [Thread-1941 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1941. 2024-11-11T22:27:51,748 DEBUG [Thread-1941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:51,748 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:51,749 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:51,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:51,765 INFO [Thread-1949 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1949. 2024-11-11T22:27:51,765 DEBUG [Thread-1949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:27:51,765 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:51,765 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:51,765 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1337 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4001 rows 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1338 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4014 rows 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1340 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4012 rows 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1340 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4020 rows 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1336 2024-11-11T22:27:51,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4008 rows 2024-11-11T22:27:51,766 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:27:51,766 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:27:51,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:51,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:51,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:27:51,767 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:27:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:27:51,772 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364071772"}]},"ts":"1731364071772"} 2024-11-11T22:27:51,774 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:27:51,774 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:27:51,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:27:51,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=324e6f269deb5f32d638f5ef06ba3bab, UNASSIGN}] 2024-11-11T22:27:51,778 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=324e6f269deb5f32d638f5ef06ba3bab, UNASSIGN 2024-11-11T22:27:51,779 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=324e6f269deb5f32d638f5ef06ba3bab, regionState=CLOSING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:51,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=324e6f269deb5f32d638f5ef06ba3bab, UNASSIGN because future has completed 2024-11-11T22:27:51,781 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:27:51,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:27:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:27:51,934 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:51,934 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:27:51,934 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing 324e6f269deb5f32d638f5ef06ba3bab, disabling compactions & flushes 2024-11-11T22:27:51,934 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:51,934 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:51,934 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. after waiting 0 ms 2024-11-11T22:27:51,934 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:51,935 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing 324e6f269deb5f32d638f5ef06ba3bab 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T22:27:51,939 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/611019bbc740451cb39ccb25564849b9 is 50, key is test_row_0/A:col10/1731364063123/Put/seqid=0 2024-11-11T22:27:51,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742340_1516 (size=12301) 2024-11-11T22:27:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:27:52,345 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/611019bbc740451cb39ccb25564849b9 2024-11-11T22:27:52,357 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6d8b7685ea3541c6b6ae4a131761f9b0 is 50, key is test_row_0/B:col10/1731364063123/Put/seqid=0 2024-11-11T22:27:52,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742341_1517 (size=12301) 2024-11-11T22:27:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:27:52,763 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6d8b7685ea3541c6b6ae4a131761f9b0 2024-11-11T22:27:52,776 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/bbc5e28d51774d0a9713647c6494a7cb is 50, key is test_row_0/C:col10/1731364063123/Put/seqid=0 2024-11-11T22:27:52,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742342_1518 (size=12301) 2024-11-11T22:27:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:27:53,181 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/bbc5e28d51774d0a9713647c6494a7cb 2024-11-11T22:27:53,187 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/A/611019bbc740451cb39ccb25564849b9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/611019bbc740451cb39ccb25564849b9 2024-11-11T22:27:53,191 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/611019bbc740451cb39ccb25564849b9, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T22:27:53,192 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/B/6d8b7685ea3541c6b6ae4a131761f9b0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6d8b7685ea3541c6b6ae4a131761f9b0 2024-11-11T22:27:53,197 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6d8b7685ea3541c6b6ae4a131761f9b0, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T22:27:53,198 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/.tmp/C/bbc5e28d51774d0a9713647c6494a7cb as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/bbc5e28d51774d0a9713647c6494a7cb 2024-11-11T22:27:53,201 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/bbc5e28d51774d0a9713647c6494a7cb, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T22:27:53,202 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 324e6f269deb5f32d638f5ef06ba3bab in 1268ms, sequenceid=396, compaction requested=true 2024-11-11T22:27:53,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2b9fc003359645cea05d370b423498d4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/278f0ccc3c5741529cd1d18683e1ae5b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a4a63cddc1bf4df58eaeb4b02f9a4b21, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c3092a9f81ec484fbb5d36185060ff25, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/590e8a2c94d34ab8a8506d93e44fddc7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/27399c6de50547afaf1840e986c7aa8e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/23d72ebd4c58404a83213daf68299b3e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/d44353d4adc949069bc4736095c390a0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/adc59025ec6b4572ae3427fb3993b3be, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/bffb1f7baf1f4d6283f16bbf261d6098, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/645d8ca516cc4317a4e7de91db5ccff4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/b97ccdd3e405484685c60484ac01bb25, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/f9bd12d9b06f443984cad5e62574f0f2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/3958081260c8407f83ea69a04d9663de, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/98d1b161fc4240f9982f69649af670bb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a097ba7f52d547f28e3e3e9d50a671fd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/344e745dcfd54e0b841c54f8e47fe953, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/384d54048d42446ca125270626077866, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/8e800f73b7524f85aa66f3fd5f5e87b9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2e0e0deb8f1f466896beef1eb379fbc4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/b539bc48a19546fea8c8ae2f3ea48c2b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/11831260519945c597b4071bdcaba71a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a9adc40dc7e34c72aa82e216b56f2be8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c2ab73b5d462437eaf3a93ceb57484e0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/7f8c1d14941b4aea8df4099a84f29ea7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/94d756717fe74abdad96f8b87b09a4fd] to archive 2024-11-11T22:27:53,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:27:53,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2b9fc003359645cea05d370b423498d4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2b9fc003359645cea05d370b423498d4 2024-11-11T22:27:53,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/278f0ccc3c5741529cd1d18683e1ae5b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/278f0ccc3c5741529cd1d18683e1ae5b 2024-11-11T22:27:53,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a4a63cddc1bf4df58eaeb4b02f9a4b21 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a4a63cddc1bf4df58eaeb4b02f9a4b21 2024-11-11T22:27:53,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c3092a9f81ec484fbb5d36185060ff25 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c3092a9f81ec484fbb5d36185060ff25 2024-11-11T22:27:53,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/590e8a2c94d34ab8a8506d93e44fddc7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/590e8a2c94d34ab8a8506d93e44fddc7 2024-11-11T22:27:53,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/27399c6de50547afaf1840e986c7aa8e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/27399c6de50547afaf1840e986c7aa8e 2024-11-11T22:27:53,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/23d72ebd4c58404a83213daf68299b3e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/23d72ebd4c58404a83213daf68299b3e 2024-11-11T22:27:53,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/d44353d4adc949069bc4736095c390a0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/d44353d4adc949069bc4736095c390a0 2024-11-11T22:27:53,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/adc59025ec6b4572ae3427fb3993b3be to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/adc59025ec6b4572ae3427fb3993b3be 2024-11-11T22:27:53,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/bffb1f7baf1f4d6283f16bbf261d6098 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/bffb1f7baf1f4d6283f16bbf261d6098 2024-11-11T22:27:53,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/645d8ca516cc4317a4e7de91db5ccff4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/645d8ca516cc4317a4e7de91db5ccff4 2024-11-11T22:27:53,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/b97ccdd3e405484685c60484ac01bb25 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/b97ccdd3e405484685c60484ac01bb25 2024-11-11T22:27:53,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/f9bd12d9b06f443984cad5e62574f0f2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/f9bd12d9b06f443984cad5e62574f0f2 2024-11-11T22:27:53,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/3958081260c8407f83ea69a04d9663de to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/3958081260c8407f83ea69a04d9663de 2024-11-11T22:27:53,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/98d1b161fc4240f9982f69649af670bb to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/98d1b161fc4240f9982f69649af670bb 2024-11-11T22:27:53,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a097ba7f52d547f28e3e3e9d50a671fd to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a097ba7f52d547f28e3e3e9d50a671fd 2024-11-11T22:27:53,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/344e745dcfd54e0b841c54f8e47fe953 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/344e745dcfd54e0b841c54f8e47fe953 2024-11-11T22:27:53,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/384d54048d42446ca125270626077866 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/384d54048d42446ca125270626077866 2024-11-11T22:27:53,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/8e800f73b7524f85aa66f3fd5f5e87b9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/8e800f73b7524f85aa66f3fd5f5e87b9 2024-11-11T22:27:53,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2e0e0deb8f1f466896beef1eb379fbc4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/2e0e0deb8f1f466896beef1eb379fbc4 2024-11-11T22:27:53,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/b539bc48a19546fea8c8ae2f3ea48c2b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/b539bc48a19546fea8c8ae2f3ea48c2b 2024-11-11T22:27:53,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/11831260519945c597b4071bdcaba71a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/11831260519945c597b4071bdcaba71a 2024-11-11T22:27:53,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a9adc40dc7e34c72aa82e216b56f2be8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/a9adc40dc7e34c72aa82e216b56f2be8 2024-11-11T22:27:53,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c2ab73b5d462437eaf3a93ceb57484e0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/c2ab73b5d462437eaf3a93ceb57484e0 2024-11-11T22:27:53,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/7f8c1d14941b4aea8df4099a84f29ea7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/7f8c1d14941b4aea8df4099a84f29ea7 2024-11-11T22:27:53,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/94d756717fe74abdad96f8b87b09a4fd to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/94d756717fe74abdad96f8b87b09a4fd 2024-11-11T22:27:53,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/7ff4033914bf40fba9947481cf03c60f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bc55e1a90c44c0396e52ea1349b455f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ceb4841ba7ef4252bbc5e0c10d4219d5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/54bef576116241b8bee3c3036fd2c074, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/f3e2f990714b4a099aba86516adecf29, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/25be6501562f41d996baa3efdb1bdb34, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/45378cf341b6446598c5eda1c897a957, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/9625a547c5c94dcd85b2c72b3b3d9b17, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3fa60c18f2524a27a8608f6cfe622b1e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/935366e9a59d41629196f8662ceac2d6, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/803ff70eff564701a329133c0f768fa0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/309f686eb844480f8021dd5c703848c7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6ba468026fab4630beb8e5d706f055c1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/540dc3942bd14fbfb21aab596c3d4321, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ec40ed7c239d4509be7898cdedac0f5b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/57cf5fd488ac4979a68df080297b58a3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bf9fbfcd06e4759bacfabfec6bbb45c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/05d47171c4ff4c4b90e69fcd4bb7a11e, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/a53e0f7e342347e8b8d20d663cb40ba0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/b49c64a4c1fd4af7b82c226086c91d9f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3138a34676dc4ebf8c49456ca61dd438, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/fa7c6f6cf988408a954a042a009b9bbf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/cb4c5aeb8c934fcf93230ec81ca94683, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/574891adba3e4600b65432dc1577fa0a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3f8455cb2bc24600aebcbc5f9b2c829a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/60ee4ca441884639bdfb84eb73b70d65] to archive 2024-11-11T22:27:53,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:27:53,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/7ff4033914bf40fba9947481cf03c60f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/7ff4033914bf40fba9947481cf03c60f 2024-11-11T22:27:53,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bc55e1a90c44c0396e52ea1349b455f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bc55e1a90c44c0396e52ea1349b455f 2024-11-11T22:27:53,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ceb4841ba7ef4252bbc5e0c10d4219d5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ceb4841ba7ef4252bbc5e0c10d4219d5 2024-11-11T22:27:53,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/54bef576116241b8bee3c3036fd2c074 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/54bef576116241b8bee3c3036fd2c074 2024-11-11T22:27:53,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/f3e2f990714b4a099aba86516adecf29 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/f3e2f990714b4a099aba86516adecf29 2024-11-11T22:27:53,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/25be6501562f41d996baa3efdb1bdb34 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/25be6501562f41d996baa3efdb1bdb34 2024-11-11T22:27:53,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/45378cf341b6446598c5eda1c897a957 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/45378cf341b6446598c5eda1c897a957 2024-11-11T22:27:53,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/9625a547c5c94dcd85b2c72b3b3d9b17 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/9625a547c5c94dcd85b2c72b3b3d9b17 2024-11-11T22:27:53,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3fa60c18f2524a27a8608f6cfe622b1e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3fa60c18f2524a27a8608f6cfe622b1e 2024-11-11T22:27:53,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/935366e9a59d41629196f8662ceac2d6 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/935366e9a59d41629196f8662ceac2d6 2024-11-11T22:27:53,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/803ff70eff564701a329133c0f768fa0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/803ff70eff564701a329133c0f768fa0 2024-11-11T22:27:53,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/309f686eb844480f8021dd5c703848c7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/309f686eb844480f8021dd5c703848c7 2024-11-11T22:27:53,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6ba468026fab4630beb8e5d706f055c1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6ba468026fab4630beb8e5d706f055c1 2024-11-11T22:27:53,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/540dc3942bd14fbfb21aab596c3d4321 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/540dc3942bd14fbfb21aab596c3d4321 2024-11-11T22:27:53,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ec40ed7c239d4509be7898cdedac0f5b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/ec40ed7c239d4509be7898cdedac0f5b 2024-11-11T22:27:53,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/57cf5fd488ac4979a68df080297b58a3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/57cf5fd488ac4979a68df080297b58a3 2024-11-11T22:27:53,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bf9fbfcd06e4759bacfabfec6bbb45c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6bf9fbfcd06e4759bacfabfec6bbb45c 2024-11-11T22:27:53,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/05d47171c4ff4c4b90e69fcd4bb7a11e to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/05d47171c4ff4c4b90e69fcd4bb7a11e 2024-11-11T22:27:53,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/a53e0f7e342347e8b8d20d663cb40ba0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/a53e0f7e342347e8b8d20d663cb40ba0 2024-11-11T22:27:53,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/b49c64a4c1fd4af7b82c226086c91d9f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/b49c64a4c1fd4af7b82c226086c91d9f 2024-11-11T22:27:53,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3138a34676dc4ebf8c49456ca61dd438 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3138a34676dc4ebf8c49456ca61dd438 2024-11-11T22:27:53,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/fa7c6f6cf988408a954a042a009b9bbf to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/fa7c6f6cf988408a954a042a009b9bbf 2024-11-11T22:27:53,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/cb4c5aeb8c934fcf93230ec81ca94683 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/cb4c5aeb8c934fcf93230ec81ca94683 2024-11-11T22:27:53,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/574891adba3e4600b65432dc1577fa0a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/574891adba3e4600b65432dc1577fa0a 2024-11-11T22:27:53,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3f8455cb2bc24600aebcbc5f9b2c829a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/3f8455cb2bc24600aebcbc5f9b2c829a 2024-11-11T22:27:53,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/60ee4ca441884639bdfb84eb73b70d65 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/60ee4ca441884639bdfb84eb73b70d65 2024-11-11T22:27:53,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/789aece9d5da4861a477e748e6a41038, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/d77fa2f41cc043b0a8c51b623340a88b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/96b0a2c978f845859c1801e11ea2770b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f036c046b398414282952ff85ac312cb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a00fba3c06ba4ab4a84d9e8b30cdf84b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c499138e11ba4f9d9005f9dafbabaec5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dd169a2958e046f68e352494782b8d0b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f449b8f6bf99424fad788c9fc230e402, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/08410c00ea254ab4b86be4fadc0abcf6, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/60795e6ef4d6481ca026e0f00350a9ce, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/516cd2553da344ecaa78cc023ecaa85a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/beeea6a47ccb418ba488a1b482e06e5d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/e2f1c0923f474536b9fe7328f8c18c9d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c244971d8bdc436c9deedcf81ab1eedd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dfc0660bd1794dd4a490223188544bab, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a50ca3df911d44869713b0f9d5892e5d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/695e5fe2e22b4a918baafb9825780915, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/4d7d3c40ef0043d4b72c5b8128039102, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/29f8ba28491e418db93dfe4462e04ee2, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/99aab58caf574724942c220fde93c7d7, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/30630256bdec437bb6e78d23b784c0c8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/246ae43c51c54b2fbc5a06af30737ab8, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/2c8eb9128e1d448881ea196514b65100, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/5d7e033aa5804060a7d93a1bf15fe919, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/94d7261d98534907ab30767471a4a4fc, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f36151e6222f484e97b0c1d9db5be997] to archive 2024-11-11T22:27:53,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:27:53,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/789aece9d5da4861a477e748e6a41038 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/789aece9d5da4861a477e748e6a41038 2024-11-11T22:27:53,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/d77fa2f41cc043b0a8c51b623340a88b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/d77fa2f41cc043b0a8c51b623340a88b 2024-11-11T22:27:53,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/96b0a2c978f845859c1801e11ea2770b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/96b0a2c978f845859c1801e11ea2770b 2024-11-11T22:27:53,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f036c046b398414282952ff85ac312cb to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f036c046b398414282952ff85ac312cb 2024-11-11T22:27:53,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a00fba3c06ba4ab4a84d9e8b30cdf84b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a00fba3c06ba4ab4a84d9e8b30cdf84b 2024-11-11T22:27:53,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c499138e11ba4f9d9005f9dafbabaec5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c499138e11ba4f9d9005f9dafbabaec5 2024-11-11T22:27:53,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dd169a2958e046f68e352494782b8d0b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dd169a2958e046f68e352494782b8d0b 2024-11-11T22:27:53,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f449b8f6bf99424fad788c9fc230e402 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f449b8f6bf99424fad788c9fc230e402 2024-11-11T22:27:53,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/08410c00ea254ab4b86be4fadc0abcf6 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/08410c00ea254ab4b86be4fadc0abcf6 2024-11-11T22:27:53,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/60795e6ef4d6481ca026e0f00350a9ce to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/60795e6ef4d6481ca026e0f00350a9ce 2024-11-11T22:27:53,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/516cd2553da344ecaa78cc023ecaa85a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/516cd2553da344ecaa78cc023ecaa85a 2024-11-11T22:27:53,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/beeea6a47ccb418ba488a1b482e06e5d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/beeea6a47ccb418ba488a1b482e06e5d 2024-11-11T22:27:53,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/e2f1c0923f474536b9fe7328f8c18c9d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/e2f1c0923f474536b9fe7328f8c18c9d 2024-11-11T22:27:53,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c244971d8bdc436c9deedcf81ab1eedd to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/c244971d8bdc436c9deedcf81ab1eedd 2024-11-11T22:27:53,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dfc0660bd1794dd4a490223188544bab to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/dfc0660bd1794dd4a490223188544bab 2024-11-11T22:27:53,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a50ca3df911d44869713b0f9d5892e5d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/a50ca3df911d44869713b0f9d5892e5d 2024-11-11T22:27:53,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/695e5fe2e22b4a918baafb9825780915 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/695e5fe2e22b4a918baafb9825780915 2024-11-11T22:27:53,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/4d7d3c40ef0043d4b72c5b8128039102 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/4d7d3c40ef0043d4b72c5b8128039102 2024-11-11T22:27:53,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/29f8ba28491e418db93dfe4462e04ee2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/29f8ba28491e418db93dfe4462e04ee2 2024-11-11T22:27:53,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/99aab58caf574724942c220fde93c7d7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/99aab58caf574724942c220fde93c7d7 2024-11-11T22:27:53,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/30630256bdec437bb6e78d23b784c0c8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/30630256bdec437bb6e78d23b784c0c8 2024-11-11T22:27:53,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/246ae43c51c54b2fbc5a06af30737ab8 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/246ae43c51c54b2fbc5a06af30737ab8 2024-11-11T22:27:53,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/2c8eb9128e1d448881ea196514b65100 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/2c8eb9128e1d448881ea196514b65100 2024-11-11T22:27:53,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/5d7e033aa5804060a7d93a1bf15fe919 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/5d7e033aa5804060a7d93a1bf15fe919 2024-11-11T22:27:53,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/94d7261d98534907ab30767471a4a4fc to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/94d7261d98534907ab30767471a4a4fc 2024-11-11T22:27:53,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f36151e6222f484e97b0c1d9db5be997 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/f36151e6222f484e97b0c1d9db5be997 2024-11-11T22:27:53,265 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/recovered.edits/399.seqid, newMaxSeqId=399, maxSeqId=1 2024-11-11T22:27:53,266 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. 2024-11-11T22:27:53,266 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for 324e6f269deb5f32d638f5ef06ba3bab: Waiting for close lock at 1731364071934Running coprocessor pre-close hooks at 1731364071934Disabling compacts and flushes for region at 1731364071934Disabling writes for close at 1731364071934Obtaining lock to block concurrent updates at 1731364071935 (+1 ms)Preparing flush snapshotting stores in 324e6f269deb5f32d638f5ef06ba3bab at 1731364071935Finished memstore snapshotting TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731364071935Flushing stores of TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab. at 1731364071935Flushing 324e6f269deb5f32d638f5ef06ba3bab/A: creating writer at 1731364071936 (+1 ms)Flushing 324e6f269deb5f32d638f5ef06ba3bab/A: appending metadata at 1731364071939 (+3 ms)Flushing 324e6f269deb5f32d638f5ef06ba3bab/A: closing flushed file at 1731364071939Flushing 324e6f269deb5f32d638f5ef06ba3bab/B: creating writer at 1731364072353 (+414 ms)Flushing 324e6f269deb5f32d638f5ef06ba3bab/B: appending metadata at 1731364072357 (+4 ms)Flushing 324e6f269deb5f32d638f5ef06ba3bab/B: closing flushed file at 1731364072357Flushing 324e6f269deb5f32d638f5ef06ba3bab/C: creating writer at 1731364072771 (+414 ms)Flushing 324e6f269deb5f32d638f5ef06ba3bab/C: appending metadata at 1731364072775 (+4 ms)Flushing 324e6f269deb5f32d638f5ef06ba3bab/C: closing flushed file at 1731364072775Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5248d62b: reopening flushed file at 1731364073186 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e7a1920: reopening flushed file at 1731364073191 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8e9b7a5: reopening flushed file at 1731364073197 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 324e6f269deb5f32d638f5ef06ba3bab in 1268ms, sequenceid=396, compaction requested=true at 1731364073202 (+5 ms)Writing region close event to WAL at 1731364073262 (+60 ms)Running coprocessor post-close hooks at 1731364073266 (+4 ms)Closed at 1731364073266 2024-11-11T22:27:53,267 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed 324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:53,268 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=324e6f269deb5f32d638f5ef06ba3bab, regionState=CLOSED 2024-11-11T22:27:53,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:27:53,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T22:27:53,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure 324e6f269deb5f32d638f5ef06ba3bab, server=8d82b89dd27d,39613,1731363918313 in 1.4890 sec 2024-11-11T22:27:53,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-11T22:27:53,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=324e6f269deb5f32d638f5ef06ba3bab, UNASSIGN in 1.4940 sec 2024-11-11T22:27:53,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T22:27:53,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4980 sec 2024-11-11T22:27:53,275 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364073275"}]},"ts":"1731364073275"} 2024-11-11T22:27:53,277 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:27:53,277 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:27:53,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5100 sec 2024-11-11T22:27:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:27:53,915 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:53,917 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:27:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:53,922 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T22:27:53,923 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:53,925 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:53,928 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/recovered.edits] 2024-11-11T22:27:53,931 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/5706b6cb695945959fa020ff87b83f75 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/5706b6cb695945959fa020ff87b83f75 2024-11-11T22:27:53,932 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/611019bbc740451cb39ccb25564849b9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/611019bbc740451cb39ccb25564849b9 2024-11-11T22:27:53,933 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/9d92f07fa07d47539911b6ea3af1014a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/A/9d92f07fa07d47539911b6ea3af1014a 2024-11-11T22:27:53,936 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/254de0d378b84772aae2975884d62c62 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/254de0d378b84772aae2975884d62c62 2024-11-11T22:27:53,937 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/52f546625a994063997becd8e92501c9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/52f546625a994063997becd8e92501c9 2024-11-11T22:27:53,939 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6d8b7685ea3541c6b6ae4a131761f9b0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/B/6d8b7685ea3541c6b6ae4a131761f9b0 2024-11-11T22:27:53,941 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/7e8b9127f4dd472c9b817be5f308f716 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/7e8b9127f4dd472c9b817be5f308f716 2024-11-11T22:27:53,943 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/b9b0a353d4824f02a4e240ad4c67b33d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/b9b0a353d4824f02a4e240ad4c67b33d 2024-11-11T22:27:53,944 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/bbc5e28d51774d0a9713647c6494a7cb to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/C/bbc5e28d51774d0a9713647c6494a7cb 2024-11-11T22:27:53,947 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/recovered.edits/399.seqid to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab/recovered.edits/399.seqid 2024-11-11T22:27:53,948 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/324e6f269deb5f32d638f5ef06ba3bab 2024-11-11T22:27:53,948 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:27:53,951 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:53,953 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:27:53,956 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:27:53,957 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:53,957 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:27:53,958 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731364073957"}]},"ts":"9223372036854775807"} 2024-11-11T22:27:53,960 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:27:53,960 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 324e6f269deb5f32d638f5ef06ba3bab, NAME => 'TestAcidGuarantees,,1731364040481.324e6f269deb5f32d638f5ef06ba3bab.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:27:53,960 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:27:53,960 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731364073960"}]},"ts":"9223372036854775807"} 2024-11-11T22:27:53,963 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:27:53,963 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:53,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-11T22:27:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T22:27:54,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:27:54,037 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:54,048 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 241), OpenFileDescriptor=443 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=444 (was 475), ProcessCount=11 (was 11), AvailableMemoryMB=5431 (was 5461) 2024-11-11T22:27:54,056 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=444, ProcessCount=11, AvailableMemoryMB=5431 2024-11-11T22:27:54,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:27:54,058 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:27:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:54,060 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:27:54,060 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:54,060 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-11-11T22:27:54,061 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:27:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T22:27:54,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742343_1519 (size=959) 2024-11-11T22:27:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T22:27:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T22:27:54,471 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e 2024-11-11T22:27:54,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742344_1520 (size=53) 2024-11-11T22:27:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T22:27:54,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:27:54,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 86be86bb450634ab24cca4850cb6c7ec, disabling compactions & flushes 2024-11-11T22:27:54,885 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:54,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:54,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. after waiting 0 ms 2024-11-11T22:27:54,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:54,886 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:54,886 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 86be86bb450634ab24cca4850cb6c7ec: Waiting for close lock at 1731364074885Disabling compacts and flushes for region at 1731364074885Disabling writes for close at 1731364074885Writing region close event to WAL at 1731364074886 (+1 ms)Closed at 1731364074886 2024-11-11T22:27:54,889 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:27:54,889 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731364074889"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731364074889"}]},"ts":"1731364074889"} 2024-11-11T22:27:54,892 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:27:54,893 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:27:54,893 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364074893"}]},"ts":"1731364074893"} 2024-11-11T22:27:54,894 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:27:54,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, ASSIGN}] 2024-11-11T22:27:54,896 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, ASSIGN 2024-11-11T22:27:54,897 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, ASSIGN; state=OFFLINE, location=8d82b89dd27d,39613,1731363918313; forceNewPlan=false, retain=false 2024-11-11T22:27:55,048 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=86be86bb450634ab24cca4850cb6c7ec, regionState=OPENING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:55,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, ASSIGN because future has completed 2024-11-11T22:27:55,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:27:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T22:27:55,213 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:55,213 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:27:55,213 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,213 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:27:55,214 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,214 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,215 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,217 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be86bb450634ab24cca4850cb6c7ec columnFamilyName A 2024-11-11T22:27:55,217 DEBUG [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:55,218 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(327): Store=86be86bb450634ab24cca4850cb6c7ec/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:27:55,218 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,219 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be86bb450634ab24cca4850cb6c7ec columnFamilyName B 2024-11-11T22:27:55,219 DEBUG [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:55,219 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(327): Store=86be86bb450634ab24cca4850cb6c7ec/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:27:55,219 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,220 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be86bb450634ab24cca4850cb6c7ec columnFamilyName C 2024-11-11T22:27:55,220 DEBUG [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:55,221 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(327): Store=86be86bb450634ab24cca4850cb6c7ec/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:27:55,221 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,221 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,222 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,223 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,223 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,223 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:27:55,224 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,225 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:27:55,226 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened 86be86bb450634ab24cca4850cb6c7ec; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59593226, jitterRate=-0.11199173331260681}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:27:55,226 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:55,226 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for 86be86bb450634ab24cca4850cb6c7ec: Running coprocessor pre-open hook at 1731364075214Writing region info on filesystem at 1731364075214Initializing all the Stores at 1731364075215 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364075215Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364075215Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364075215Cleaning up temporary data from old regions at 1731364075223 (+8 ms)Running coprocessor post-open hooks at 1731364075226 (+3 ms)Region opened successfully at 1731364075226 2024-11-11T22:27:55,227 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., pid=136, masterSystemTime=1731364075206 2024-11-11T22:27:55,228 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:55,228 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:55,229 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=86be86bb450634ab24cca4850cb6c7ec, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:55,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:27:55,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-11T22:27:55,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 in 179 msec 2024-11-11T22:27:55,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-11T22:27:55,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, ASSIGN in 337 msec 2024-11-11T22:27:55,234 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:27:55,234 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364075234"}]},"ts":"1731364075234"} 2024-11-11T22:27:55,235 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:27:55,236 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:27:55,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-11T22:27:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-11T22:27:56,204 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:56,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:56,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:56,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:56,207 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f3d8819, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:56,210 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:56,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a42efb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:56,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:56,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:56,216 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:27:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,40133,1731363917560 2024-11-11T22:27:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47ebf922 2024-11-11T22:27:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:27:56,218 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:27:56,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:27:56,220 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:27:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T22:27:56,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742345_1521 (size=995) 2024-11-11T22:27:56,635 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T22:27:56,635 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T22:27:56,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:27:56,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, REOPEN/MOVE}] 2024-11-11T22:27:56,644 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, REOPEN/MOVE 2024-11-11T22:27:56,645 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=86be86bb450634ab24cca4850cb6c7ec, regionState=CLOSING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:56,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, REOPEN/MOVE because future has completed 2024-11-11T22:27:56,647 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:27:56,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:27:56,800 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:56,800 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:27:56,800 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 86be86bb450634ab24cca4850cb6c7ec, disabling compactions & flushes 2024-11-11T22:27:56,800 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:56,800 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:56,800 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. after waiting 0 ms 2024-11-11T22:27:56,801 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:56,808 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T22:27:56,809 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:56,810 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 86be86bb450634ab24cca4850cb6c7ec: Waiting for close lock at 1731364076800Running coprocessor pre-close hooks at 1731364076800Disabling compacts and flushes for region at 1731364076800Disabling writes for close at 1731364076800Writing region close event to WAL at 1731364076801 (+1 ms)Running coprocessor post-close hooks at 1731364076809 (+8 ms)Closed at 1731364076809 2024-11-11T22:27:56,810 WARN [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: 86be86bb450634ab24cca4850cb6c7ec to self. 2024-11-11T22:27:56,812 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:56,813 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=86be86bb450634ab24cca4850cb6c7ec, regionState=CLOSED 2024-11-11T22:27:56,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:27:56,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T22:27:56,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 in 168 msec 2024-11-11T22:27:56,818 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, REOPEN/MOVE; state=CLOSED, location=8d82b89dd27d,39613,1731363918313; forceNewPlan=false, retain=true 2024-11-11T22:27:56,968 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=86be86bb450634ab24cca4850cb6c7ec, regionState=OPENING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:56,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, REOPEN/MOVE because future has completed 2024-11-11T22:27:56,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:27:57,132 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,132 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:27:57,133 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,133 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:27:57,133 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,133 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,135 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,136 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be86bb450634ab24cca4850cb6c7ec columnFamilyName A 2024-11-11T22:27:57,137 DEBUG [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:57,138 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(327): Store=86be86bb450634ab24cca4850cb6c7ec/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:27:57,138 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,139 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be86bb450634ab24cca4850cb6c7ec columnFamilyName B 2024-11-11T22:27:57,139 DEBUG [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:57,140 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(327): Store=86be86bb450634ab24cca4850cb6c7ec/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:27:57,140 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,140 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be86bb450634ab24cca4850cb6c7ec columnFamilyName C 2024-11-11T22:27:57,140 DEBUG [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:57,141 INFO [StoreOpener-86be86bb450634ab24cca4850cb6c7ec-1 {}] regionserver.HStore(327): Store=86be86bb450634ab24cca4850cb6c7ec/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:27:57,141 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,142 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,143 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,145 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,145 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,146 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:27:57,148 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,149 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened 86be86bb450634ab24cca4850cb6c7ec; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62043038, jitterRate=-0.0754866898059845}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:27:57,149 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,150 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for 86be86bb450634ab24cca4850cb6c7ec: Running coprocessor pre-open hook at 1731364077133Writing region info on filesystem at 1731364077133Initializing all the Stores at 1731364077134 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364077134Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364077135 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731364077135Cleaning up temporary data from old regions at 1731364077145 (+10 ms)Running coprocessor post-open hooks at 1731364077149 (+4 ms)Region opened successfully at 1731364077150 (+1 ms) 2024-11-11T22:27:57,151 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., pid=141, masterSystemTime=1731364077125 2024-11-11T22:27:57,153 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,153 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,154 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=86be86bb450634ab24cca4850cb6c7ec, regionState=OPEN, openSeqNum=5, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:27:57,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-11-11T22:27:57,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 in 186 msec 2024-11-11T22:27:57,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-11T22:27:57,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, REOPEN/MOVE in 516 msec 2024-11-11T22:27:57,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T22:27:57,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 582 msec 2024-11-11T22:27:57,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0090 sec 2024-11-11T22:27:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T22:27:57,233 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:27:57,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,234 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:57,234 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:57,235 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71e2f496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:57,236 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59abe0f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:57,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:57,238 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d37e43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:57,240 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f77f34c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:57,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:57,242 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b373287, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:57,243 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1830e75b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:57,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:57,246 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d7192e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:57,248 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:57,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a341f46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:57,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:57,251 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee849c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:57,253 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:57,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@470e49f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:57,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:57,255 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:57,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@731be3b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:57,257 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a0e0d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:57,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:57,258 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3192bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:57,260 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6333dffb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:57,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:57,261 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58bf21b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:57,263 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d64ab50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:57,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:57,264 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:57,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:57,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:57,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4273148f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:57,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:57,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:57,265 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc17759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,40133,-1 for getting cluster id 2024-11-11T22:27:57,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:27:57,267 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe' 2024-11-11T22:27:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:27:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd45ba5b-aabc-452c-bfda-cf4e8ea6bebe" 2024-11-11T22:27:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6374ab12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,40133,-1] 2024-11-11T22:27:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:27:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:27:57,268 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:27:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3050f712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:27:57,269 DEBUG [Thread-2344 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:57,269 DEBUG [Thread-2348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:57,269 DEBUG [Thread-2342 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:57,269 DEBUG [Thread-2333 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:57,270 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:27:57,270 DEBUG [Thread-2340 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:57,270 DEBUG [Thread-2337 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-11T22:27:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:57,271 DEBUG [Thread-2329 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:57,271 DEBUG [Thread-2346 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:57,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:57,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:57,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:57,272 DEBUG [Thread-2331 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:57,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:57,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:27:57,273 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:27:57,273 DEBUG [Thread-2335 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:27:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:57,274 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:27:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:27:57,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:27:57,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:57,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:27:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:27:57,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,39613,1731363918313, seqNum=-1] 2024-11-11T22:27:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:27:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:27:57,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:27:57,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:27:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:27:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:27:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:27:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:27:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5] 2024-11-11T22:27:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,283 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:27:57,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T22:27:57,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364137304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111189c868223e24e3aa2d35f458e7452df_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364077283/Put/seqid=0 2024-11-11T22:27:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364137306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364137307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742346_1522 (size=14594) 2024-11-11T22:27:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364137308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364137308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,313 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,316 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111189c868223e24e3aa2d35f458e7452df_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111189c868223e24e3aa2d35f458e7452df_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:57,317 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/1cfdc166559a45c4af95d2649fe69f13, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:27:57,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/1cfdc166559a45c4af95d2649fe69f13 is 175, key is test_row_0/A:col10/1731364077283/Put/seqid=0 2024-11-11T22:27:57,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742347_1523 (size=39549) 2024-11-11T22:27:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:27:57,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364137413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364137414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364137414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364137423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364137423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T22:27:57,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:27:57,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T22:27:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:27:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:27:57,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364137623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364137623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364137623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364137633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364137633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,726 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/1cfdc166559a45c4af95d2649fe69f13 2024-11-11T22:27:57,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T22:27:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:27:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/fec234d0c94c403dbf5980bce25b2559 is 50, key is test_row_0/B:col10/1731364077283/Put/seqid=0 2024-11-11T22:27:57,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742348_1524 (size=12001) 2024-11-11T22:27:57,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T22:27:57,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:27:57,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:57,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:27:57,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364137933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364137933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364137933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364137943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364137943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:58,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T22:27:58,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:58,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:27:58,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:58,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/fec234d0c94c403dbf5980bce25b2559 2024-11-11T22:27:58,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/38815b8d0f73439b98df2e066e1279e3 is 50, key is test_row_0/C:col10/1731364077283/Put/seqid=0 2024-11-11T22:27:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742349_1525 (size=12001) 2024-11-11T22:27:58,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T22:27:58,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:58,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:27:58,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:58,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T22:27:58,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:58,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:27:58,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:58,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:27:58,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364138443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:58,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:58,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:58,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:58,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364138443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:58,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:58,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364138443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:58,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:58,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:58,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:58,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:58,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:58,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364138453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:58,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364138453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:58,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:58,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T22:27:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:27:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:58,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:27:58,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/38815b8d0f73439b98df2e066e1279e3 2024-11-11T22:27:58,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/1cfdc166559a45c4af95d2649fe69f13 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/1cfdc166559a45c4af95d2649fe69f13 2024-11-11T22:27:58,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/1cfdc166559a45c4af95d2649fe69f13, entries=200, sequenceid=17, filesize=38.6 K 2024-11-11T22:27:58,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/fec234d0c94c403dbf5980bce25b2559 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/fec234d0c94c403dbf5980bce25b2559 2024-11-11T22:27:58,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/fec234d0c94c403dbf5980bce25b2559, entries=150, sequenceid=17, filesize=11.7 K 2024-11-11T22:27:58,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/38815b8d0f73439b98df2e066e1279e3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/38815b8d0f73439b98df2e066e1279e3 2024-11-11T22:27:58,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/38815b8d0f73439b98df2e066e1279e3, entries=150, sequenceid=17, filesize=11.7 K 2024-11-11T22:27:58,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 86be86bb450634ab24cca4850cb6c7ec in 1310ms, sequenceid=17, compaction requested=false 2024-11-11T22:27:58,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:27:58,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T22:27:58,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:58,647 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T22:27:58,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c9da996816bd4ed68ad85cc8d2d6853f_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364077307/Put/seqid=0 2024-11-11T22:27:58,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742350_1526 (size=12154) 2024-11-11T22:27:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:27:58,660 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c9da996816bd4ed68ad85cc8d2d6853f_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c9da996816bd4ed68ad85cc8d2d6853f_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:58,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/2849b6e0d2cc4525bca5350bcc5d486f, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:27:58,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/2849b6e0d2cc4525bca5350bcc5d486f is 175, key is test_row_0/A:col10/1731364077307/Put/seqid=0 2024-11-11T22:27:58,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742351_1527 (size=30955) 2024-11-11T22:27:58,782 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:27:59,065 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/2849b6e0d2cc4525bca5350bcc5d486f 2024-11-11T22:27:59,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/da4efa9880974ee9bd6ed40cd1dc29fe is 50, key is test_row_0/B:col10/1731364077307/Put/seqid=0 2024-11-11T22:27:59,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742352_1528 (size=12001) 2024-11-11T22:27:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:27:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:27:59,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:27:59,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364139469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,473 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/da4efa9880974ee9bd6ed40cd1dc29fe 2024-11-11T22:27:59,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364139471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364139471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364139472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364139472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/dfa229024ef0439db91448cd0aec56fb is 50, key is test_row_0/C:col10/1731364077307/Put/seqid=0 2024-11-11T22:27:59,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742353_1529 (size=12001) 2024-11-11T22:27:59,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364139583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364139583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364139583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364139583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364139583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364139793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364139793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364139793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364139793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:27:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364139793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:27:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:27:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:27:59,882 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/dfa229024ef0439db91448cd0aec56fb 2024-11-11T22:27:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/2849b6e0d2cc4525bca5350bcc5d486f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/2849b6e0d2cc4525bca5350bcc5d486f 2024-11-11T22:27:59,888 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/2849b6e0d2cc4525bca5350bcc5d486f, entries=150, sequenceid=40, filesize=30.2 K 2024-11-11T22:27:59,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/da4efa9880974ee9bd6ed40cd1dc29fe as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/da4efa9880974ee9bd6ed40cd1dc29fe 2024-11-11T22:27:59,891 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/da4efa9880974ee9bd6ed40cd1dc29fe, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T22:27:59,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/dfa229024ef0439db91448cd0aec56fb as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/dfa229024ef0439db91448cd0aec56fb 2024-11-11T22:27:59,894 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/dfa229024ef0439db91448cd0aec56fb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T22:27:59,895 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 86be86bb450634ab24cca4850cb6c7ec in 1248ms, sequenceid=40, compaction requested=false 2024-11-11T22:27:59,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:27:59,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:27:59,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-11T22:27:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-11T22:27:59,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-11T22:27:59,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6230 sec 2024-11-11T22:27:59,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.6290 sec 2024-11-11T22:28:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:00,105 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:28:00,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:28:00,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bb8ffa2c51594bd68a272aaa8b94cd3a_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364080105/Put/seqid=0 2024-11-11T22:28:00,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742354_1530 (size=12154) 2024-11-11T22:28:00,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364140120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364140125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364140125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364140126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364140126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364140233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364140233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364140233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364140234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364140234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364140443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364140443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364140444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364140444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364140444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:00,517 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bb8ffa2c51594bd68a272aaa8b94cd3a_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bb8ffa2c51594bd68a272aaa8b94cd3a_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:00,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/334508f76c0748828a914b5006a3776c, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:00,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/334508f76c0748828a914b5006a3776c is 175, key is test_row_0/A:col10/1731364080105/Put/seqid=0 2024-11-11T22:28:00,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742355_1531 (size=30955) 2024-11-11T22:28:00,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364140753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364140753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364140753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364140753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364140753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:00,922 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/334508f76c0748828a914b5006a3776c 2024-11-11T22:28:00,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/360c0d13fe88421e8f12f12406498951 is 50, key is test_row_0/B:col10/1731364080105/Put/seqid=0 2024-11-11T22:28:00,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742356_1532 (size=12001) 2024-11-11T22:28:01,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364141263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:01,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364141264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:01,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364141264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:01,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364141264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:01,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364141264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:01,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/360c0d13fe88421e8f12f12406498951 2024-11-11T22:28:01,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/f108bd8c470c42d6813412fe6ed55c86 is 50, key is test_row_0/C:col10/1731364080105/Put/seqid=0 2024-11-11T22:28:01,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742357_1533 (size=12001) 2024-11-11T22:28:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:28:01,434 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:28:01,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:28:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-11T22:28:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T22:28:01,436 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:28:01,437 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:28:01,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:28:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T22:28:01,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T22:28:01,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:01,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:01,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:01,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:01,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:01,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T22:28:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:01,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T22:28:01,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/f108bd8c470c42d6813412fe6ed55c86 2024-11-11T22:28:01,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/334508f76c0748828a914b5006a3776c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/334508f76c0748828a914b5006a3776c 2024-11-11T22:28:01,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/334508f76c0748828a914b5006a3776c, entries=150, sequenceid=56, filesize=30.2 K 2024-11-11T22:28:01,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/360c0d13fe88421e8f12f12406498951 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/360c0d13fe88421e8f12f12406498951 2024-11-11T22:28:01,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/360c0d13fe88421e8f12f12406498951, entries=150, sequenceid=56, filesize=11.7 K 2024-11-11T22:28:01,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/f108bd8c470c42d6813412fe6ed55c86 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/f108bd8c470c42d6813412fe6ed55c86 2024-11-11T22:28:01,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/f108bd8c470c42d6813412fe6ed55c86, entries=150, sequenceid=56, filesize=11.7 K 2024-11-11T22:28:01,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 86be86bb450634ab24cca4850cb6c7ec in 1675ms, sequenceid=56, compaction requested=true 2024-11-11T22:28:01,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:01,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:28:01,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:01,780 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:28:01,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:28:01,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:01,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:28:01,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:28:01,780 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:28:01,781 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:28:01,781 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/A is initiating minor compaction (all files) 2024-11-11T22:28:01,781 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:28:01,781 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/A in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:01,781 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/B is initiating minor compaction (all files) 2024-11-11T22:28:01,781 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/1cfdc166559a45c4af95d2649fe69f13, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/2849b6e0d2cc4525bca5350bcc5d486f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/334508f76c0748828a914b5006a3776c] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=99.1 K 2024-11-11T22:28:01,781 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/B in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:01,781 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:01,781 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/fec234d0c94c403dbf5980bce25b2559, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/da4efa9880974ee9bd6ed40cd1dc29fe, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/360c0d13fe88421e8f12f12406498951] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=35.2 K 2024-11-11T22:28:01,781 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/1cfdc166559a45c4af95d2649fe69f13, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/2849b6e0d2cc4525bca5350bcc5d486f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/334508f76c0748828a914b5006a3776c] 2024-11-11T22:28:01,781 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting fec234d0c94c403dbf5980bce25b2559, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731364077282 2024-11-11T22:28:01,781 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cfdc166559a45c4af95d2649fe69f13, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731364077281 2024-11-11T22:28:01,783 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2849b6e0d2cc4525bca5350bcc5d486f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731364077304 2024-11-11T22:28:01,783 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting da4efa9880974ee9bd6ed40cd1dc29fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731364077304 2024-11-11T22:28:01,783 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 360c0d13fe88421e8f12f12406498951, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731364080104 2024-11-11T22:28:01,783 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 334508f76c0748828a914b5006a3776c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731364080104 2024-11-11T22:28:01,789 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:01,791 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111a6449bb0494347e2b276b2ff43e6dedf_86be86bb450634ab24cca4850cb6c7ec store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:01,791 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#B#compaction#457 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:01,792 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/2958804b414248868260cc23637ea097 is 50, key is test_row_0/B:col10/1731364080105/Put/seqid=0 2024-11-11T22:28:01,794 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111a6449bb0494347e2b276b2ff43e6dedf_86be86bb450634ab24cca4850cb6c7ec, store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:01,794 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a6449bb0494347e2b276b2ff43e6dedf_86be86bb450634ab24cca4850cb6c7ec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:01,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742358_1534 (size=12104) 2024-11-11T22:28:01,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742359_1535 (size=4469) 2024-11-11T22:28:01,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T22:28:01,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:01,895 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T22:28:01,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b2484d59d9b144a2a78f947b15951e9a_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364080124/Put/seqid=0 2024-11-11T22:28:01,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742360_1536 (size=12154) 2024-11-11T22:28:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T22:28:02,207 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/2958804b414248868260cc23637ea097 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2958804b414248868260cc23637ea097 2024-11-11T22:28:02,210 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#A#compaction#456 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:02,211 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/c4a4c6b89b1b4662a378ff9d023387f4 is 175, key is test_row_0/A:col10/1731364080105/Put/seqid=0 2024-11-11T22:28:02,211 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/B of 86be86bb450634ab24cca4850cb6c7ec into 2958804b414248868260cc23637ea097(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:02,211 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:02,211 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/B, priority=13, startTime=1731364081780; duration=0sec 2024-11-11T22:28:02,211 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:02,211 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:B 2024-11-11T22:28:02,211 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:28:02,212 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:28:02,212 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/C is initiating minor compaction (all files) 2024-11-11T22:28:02,212 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/C in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:02,212 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/38815b8d0f73439b98df2e066e1279e3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/dfa229024ef0439db91448cd0aec56fb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/f108bd8c470c42d6813412fe6ed55c86] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=35.2 K 2024-11-11T22:28:02,213 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 38815b8d0f73439b98df2e066e1279e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731364077282 2024-11-11T22:28:02,213 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting dfa229024ef0439db91448cd0aec56fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731364077304 2024-11-11T22:28:02,213 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting f108bd8c470c42d6813412fe6ed55c86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731364080104 2024-11-11T22:28:02,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742361_1537 (size=31058) 2024-11-11T22:28:02,219 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/c4a4c6b89b1b4662a378ff9d023387f4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/c4a4c6b89b1b4662a378ff9d023387f4 2024-11-11T22:28:02,222 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:02,223 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/A of 86be86bb450634ab24cca4850cb6c7ec into c4a4c6b89b1b4662a378ff9d023387f4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:02,223 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:02,223 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/a5c5618b4f784b97aff932c2477daf6d is 50, key is test_row_0/C:col10/1731364080105/Put/seqid=0 2024-11-11T22:28:02,223 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/A, priority=13, startTime=1731364081780; duration=0sec 2024-11-11T22:28:02,223 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:02,223 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:A 2024-11-11T22:28:02,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742362_1538 (size=12104) 2024-11-11T22:28:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:02,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:02,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364142289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364142290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364142290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364142291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364142291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:02,308 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b2484d59d9b144a2a78f947b15951e9a_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b2484d59d9b144a2a78f947b15951e9a_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:02,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/99637cc4fb394a7fa2ad2f8ea151c4da, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/99637cc4fb394a7fa2ad2f8ea151c4da is 175, key is test_row_0/A:col10/1731364080124/Put/seqid=0 2024-11-11T22:28:02,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742363_1539 (size=30955) 2024-11-11T22:28:02,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364142403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364142403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364142404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364142403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364142404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T22:28:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364142613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364142613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364142613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364142614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364142614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,637 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/a5c5618b4f784b97aff932c2477daf6d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a5c5618b4f784b97aff932c2477daf6d 2024-11-11T22:28:02,641 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/C of 86be86bb450634ab24cca4850cb6c7ec into a5c5618b4f784b97aff932c2477daf6d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:02,641 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:02,641 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/C, priority=13, startTime=1731364081780; duration=0sec 2024-11-11T22:28:02,641 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:02,641 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:C 2024-11-11T22:28:02,712 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/99637cc4fb394a7fa2ad2f8ea151c4da 2024-11-11T22:28:02,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/69a055111bca41fd85eb35df5be9add4 is 50, key is test_row_0/B:col10/1731364080124/Put/seqid=0 2024-11-11T22:28:02,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742364_1540 (size=12001) 2024-11-11T22:28:02,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364142923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364142923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364142923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364142923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364142924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:03,121 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/69a055111bca41fd85eb35df5be9add4 2024-11-11T22:28:03,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/a46b2ca1594848b184de14f583138e01 is 50, key is test_row_0/C:col10/1731364080124/Put/seqid=0 2024-11-11T22:28:03,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742365_1541 (size=12001) 2024-11-11T22:28:03,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364143433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:03,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364143433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:03,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364143433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:03,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364143433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:03,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364143433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:03,529 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/a46b2ca1594848b184de14f583138e01 2024-11-11T22:28:03,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/99637cc4fb394a7fa2ad2f8ea151c4da as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/99637cc4fb394a7fa2ad2f8ea151c4da 2024-11-11T22:28:03,535 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/99637cc4fb394a7fa2ad2f8ea151c4da, entries=150, sequenceid=76, filesize=30.2 K 2024-11-11T22:28:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/69a055111bca41fd85eb35df5be9add4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/69a055111bca41fd85eb35df5be9add4 2024-11-11T22:28:03,539 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/69a055111bca41fd85eb35df5be9add4, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T22:28:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/a46b2ca1594848b184de14f583138e01 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a46b2ca1594848b184de14f583138e01 2024-11-11T22:28:03,542 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a46b2ca1594848b184de14f583138e01, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T22:28:03,543 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 86be86bb450634ab24cca4850cb6c7ec in 1648ms, sequenceid=76, compaction requested=false 2024-11-11T22:28:03,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:03,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:03,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-11T22:28:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-11T22:28:03,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-11T22:28:03,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1070 sec 2024-11-11T22:28:03,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.1120 sec 2024-11-11T22:28:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T22:28:03,583 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:28:03,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:28:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-11T22:28:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T22:28:03,585 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:28:03,585 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:28:03,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:28:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T22:28:03,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T22:28:03,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:03,738 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T22:28:03,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d6abdf21a43c4b35b367aa243fdcb6d8_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364082290/Put/seqid=0 2024-11-11T22:28:03,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742366_1542 (size=12154) 2024-11-11T22:28:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T22:28:04,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:04,149 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d6abdf21a43c4b35b367aa243fdcb6d8_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d6abdf21a43c4b35b367aa243fdcb6d8_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:04,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/a070faab75984a8696dbb6330cc9a32c, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:04,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/a070faab75984a8696dbb6330cc9a32c is 175, key is test_row_0/A:col10/1731364082290/Put/seqid=0 2024-11-11T22:28:04,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742367_1543 (size=30955) 2024-11-11T22:28:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T22:28:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:04,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:04,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364144463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364144464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364144465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364144466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364144466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,554 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/a070faab75984a8696dbb6330cc9a32c 2024-11-11T22:28:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/e45e309a7b144612af8368fce2708346 is 50, key is test_row_0/B:col10/1731364082290/Put/seqid=0 2024-11-11T22:28:04,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742368_1544 (size=12001) 2024-11-11T22:28:04,566 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/e45e309a7b144612af8368fce2708346 2024-11-11T22:28:04,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/444ec7f5ba294d27a951715271a77b25 is 50, key is test_row_0/C:col10/1731364082290/Put/seqid=0 2024-11-11T22:28:04,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364144573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364144573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364144573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364144573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364144573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742369_1545 (size=12001) 2024-11-11T22:28:04,581 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/444ec7f5ba294d27a951715271a77b25 2024-11-11T22:28:04,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/a070faab75984a8696dbb6330cc9a32c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a070faab75984a8696dbb6330cc9a32c 2024-11-11T22:28:04,589 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a070faab75984a8696dbb6330cc9a32c, entries=150, sequenceid=96, filesize=30.2 K 2024-11-11T22:28:04,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/e45e309a7b144612af8368fce2708346 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/e45e309a7b144612af8368fce2708346 2024-11-11T22:28:04,593 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/e45e309a7b144612af8368fce2708346, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T22:28:04,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/444ec7f5ba294d27a951715271a77b25 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/444ec7f5ba294d27a951715271a77b25 2024-11-11T22:28:04,596 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/444ec7f5ba294d27a951715271a77b25, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T22:28:04,597 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 86be86bb450634ab24cca4850cb6c7ec in 860ms, sequenceid=96, compaction requested=true 2024-11-11T22:28:04,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:04,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:04,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-11T22:28:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-11T22:28:04,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-11T22:28:04,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0130 sec 2024-11-11T22:28:04,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.0190 sec 2024-11-11T22:28:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T22:28:04,724 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:28:04,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:28:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-11T22:28:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T22:28:04,725 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:28:04,726 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:28:04,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:28:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:04,785 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:28:04,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T22:28:04,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f51da848eba649e78600fc574279964c_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364084464/Put/seqid=0 2024-11-11T22:28:04,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742370_1546 (size=14594) 2024-11-11T22:28:04,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364144799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364144799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364144800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364144800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364144801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T22:28:04,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T22:28:04,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:04,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:04,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:04,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:04,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:04,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:04,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364144913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364144913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364144914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364144914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:04,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364144914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T22:28:05,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:05,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T22:28:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364145123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364145123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364145123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364145123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364145123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T22:28:05,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:05,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,195 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:05,198 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f51da848eba649e78600fc574279964c_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f51da848eba649e78600fc574279964c_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:05,199 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/466154ba5dd6401e957c9c3535985419, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:05,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/466154ba5dd6401e957c9c3535985419 is 175, key is test_row_0/A:col10/1731364084464/Put/seqid=0 2024-11-11T22:28:05,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742371_1547 (size=39549) 2024-11-11T22:28:05,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T22:28:05,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:05,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T22:28:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364145433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364145434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364145434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364145434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364145434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T22:28:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:05,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,608 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/466154ba5dd6401e957c9c3535985419 2024-11-11T22:28:05,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/2dbb6711fbd242708477d42a374d6669 is 50, key is test_row_0/B:col10/1731364084464/Put/seqid=0 2024-11-11T22:28:05,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742372_1548 (size=12001) 2024-11-11T22:28:05,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T22:28:05,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:05,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T22:28:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T22:28:05,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364145943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364145943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364145943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364145943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364145943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:05,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:05,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:05,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:05,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:06,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/2dbb6711fbd242708477d42a374d6669 2024-11-11T22:28:06,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/15fe41f4e8a04a65af3d7ba890d1ddc1 is 50, key is test_row_0/C:col10/1731364084464/Put/seqid=0 2024-11-11T22:28:06,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742373_1549 (size=12001) 2024-11-11T22:28:06,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/15fe41f4e8a04a65af3d7ba890d1ddc1 2024-11-11T22:28:06,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/466154ba5dd6401e957c9c3535985419 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/466154ba5dd6401e957c9c3535985419 2024-11-11T22:28:06,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/466154ba5dd6401e957c9c3535985419, entries=200, sequenceid=118, filesize=38.6 K 2024-11-11T22:28:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/2dbb6711fbd242708477d42a374d6669 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2dbb6711fbd242708477d42a374d6669 2024-11-11T22:28:06,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2dbb6711fbd242708477d42a374d6669, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T22:28:06,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/15fe41f4e8a04a65af3d7ba890d1ddc1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/15fe41f4e8a04a65af3d7ba890d1ddc1 2024-11-11T22:28:06,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/15fe41f4e8a04a65af3d7ba890d1ddc1, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T22:28:06,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 86be86bb450634ab24cca4850cb6c7ec in 1255ms, sequenceid=118, compaction requested=true 2024-11-11T22:28:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:28:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:06,040 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:28:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:28:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:28:06,040 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:28:06,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:06,041 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:28:06,041 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:28:06,041 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/A is initiating minor compaction (all files) 2024-11-11T22:28:06,041 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/B is initiating minor compaction (all files) 2024-11-11T22:28:06,041 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/B in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:06,041 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/A in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:06,041 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2958804b414248868260cc23637ea097, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/69a055111bca41fd85eb35df5be9add4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/e45e309a7b144612af8368fce2708346, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2dbb6711fbd242708477d42a374d6669] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=47.0 K 2024-11-11T22:28:06,041 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/c4a4c6b89b1b4662a378ff9d023387f4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/99637cc4fb394a7fa2ad2f8ea151c4da, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a070faab75984a8696dbb6330cc9a32c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/466154ba5dd6401e957c9c3535985419] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=129.4 K 2024-11-11T22:28:06,041 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:06,041 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/c4a4c6b89b1b4662a378ff9d023387f4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/99637cc4fb394a7fa2ad2f8ea151c4da, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a070faab75984a8696dbb6330cc9a32c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/466154ba5dd6401e957c9c3535985419] 2024-11-11T22:28:06,042 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 2958804b414248868260cc23637ea097, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731364080104 2024-11-11T22:28:06,042 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a4c6b89b1b4662a378ff9d023387f4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731364080104 2024-11-11T22:28:06,042 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99637cc4fb394a7fa2ad2f8ea151c4da, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731364080124 2024-11-11T22:28:06,042 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 69a055111bca41fd85eb35df5be9add4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731364080124 2024-11-11T22:28:06,042 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting e45e309a7b144612af8368fce2708346, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731364082289 2024-11-11T22:28:06,042 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting a070faab75984a8696dbb6330cc9a32c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731364082289 2024-11-11T22:28:06,042 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dbb6711fbd242708477d42a374d6669, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731364084464 2024-11-11T22:28:06,042 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 466154ba5dd6401e957c9c3535985419, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731364084463 2024-11-11T22:28:06,047 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:06,048 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111ec3f9a8051db4bc39c14f7e5c465f4bd_86be86bb450634ab24cca4850cb6c7ec store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:06,048 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:06,049 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/a4a115d89e5947b38e8e979e3b0e4746 is 50, key is test_row_0/B:col10/1731364084464/Put/seqid=0 2024-11-11T22:28:06,050 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111ec3f9a8051db4bc39c14f7e5c465f4bd_86be86bb450634ab24cca4850cb6c7ec, store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:06,050 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ec3f9a8051db4bc39c14f7e5c465f4bd_86be86bb450634ab24cca4850cb6c7ec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:06,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742374_1550 (size=4469) 2024-11-11T22:28:06,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742375_1551 (size=12241) 2024-11-11T22:28:06,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T22:28:06,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:06,099 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T22:28:06,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5ec655a6cc34a9d87bd621e008efb7e_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364084800/Put/seqid=0 2024-11-11T22:28:06,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742376_1552 (size=12304) 2024-11-11T22:28:06,455 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#A#compaction#469 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:06,455 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/21a83c0bb7fa4e9eb48be13fbea89007 is 175, key is test_row_0/A:col10/1731364084464/Put/seqid=0 2024-11-11T22:28:06,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742377_1553 (size=31195) 2024-11-11T22:28:06,468 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/a4a115d89e5947b38e8e979e3b0e4746 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a4a115d89e5947b38e8e979e3b0e4746 2024-11-11T22:28:06,472 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/B of 86be86bb450634ab24cca4850cb6c7ec into a4a115d89e5947b38e8e979e3b0e4746(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:06,472 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:06,472 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/B, priority=12, startTime=1731364086040; duration=0sec 2024-11-11T22:28:06,472 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:06,472 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:B 2024-11-11T22:28:06,472 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:28:06,473 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:28:06,473 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/C is initiating minor compaction (all files) 2024-11-11T22:28:06,473 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/C in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:06,473 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a5c5618b4f784b97aff932c2477daf6d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a46b2ca1594848b184de14f583138e01, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/444ec7f5ba294d27a951715271a77b25, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/15fe41f4e8a04a65af3d7ba890d1ddc1] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=47.0 K 2024-11-11T22:28:06,473 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c5618b4f784b97aff932c2477daf6d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731364080104 2024-11-11T22:28:06,474 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a46b2ca1594848b184de14f583138e01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731364080124 2024-11-11T22:28:06,474 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 444ec7f5ba294d27a951715271a77b25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731364082289 2024-11-11T22:28:06,474 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 15fe41f4e8a04a65af3d7ba890d1ddc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731364084464 2024-11-11T22:28:06,480 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#C#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:06,481 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/ae6b3e8527cf4df9a4b8466e68b06284 is 50, key is test_row_0/C:col10/1731364084464/Put/seqid=0 2024-11-11T22:28:06,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742378_1554 (size=12241) 2024-11-11T22:28:06,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:06,511 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5ec655a6cc34a9d87bd621e008efb7e_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5ec655a6cc34a9d87bd621e008efb7e_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:06,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/3e1a35b4ba1145018808a058f4d87c3c, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:06,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/3e1a35b4ba1145018808a058f4d87c3c is 175, key is test_row_0/A:col10/1731364084800/Put/seqid=0 2024-11-11T22:28:06,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742379_1555 (size=31105) 2024-11-11T22:28:06,861 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/21a83c0bb7fa4e9eb48be13fbea89007 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/21a83c0bb7fa4e9eb48be13fbea89007 2024-11-11T22:28:06,864 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/A of 86be86bb450634ab24cca4850cb6c7ec into 21a83c0bb7fa4e9eb48be13fbea89007(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:06,865 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:06,865 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/A, priority=12, startTime=1731364086040; duration=0sec 2024-11-11T22:28:06,865 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:06,865 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:A 2024-11-11T22:28:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T22:28:06,887 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/ae6b3e8527cf4df9a4b8466e68b06284 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/ae6b3e8527cf4df9a4b8466e68b06284 2024-11-11T22:28:06,892 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/C of 86be86bb450634ab24cca4850cb6c7ec into ae6b3e8527cf4df9a4b8466e68b06284(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:06,892 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:06,892 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/C, priority=12, startTime=1731364086040; duration=0sec 2024-11-11T22:28:06,892 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:06,892 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:C 2024-11-11T22:28:06,915 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/3e1a35b4ba1145018808a058f4d87c3c 2024-11-11T22:28:06,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/8d3a92d3e32c43b59ae1d1940bfab03a is 50, key is test_row_0/B:col10/1731364084800/Put/seqid=0 2024-11-11T22:28:06,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742380_1556 (size=12151) 2024-11-11T22:28:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:06,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:06,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364146966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:06,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364146968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:06,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364146968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:06,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364146969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:06,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364146970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364147073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364147083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364147083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364147083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364147083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364147283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364147293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364147293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364147293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364147294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,324 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/8d3a92d3e32c43b59ae1d1940bfab03a 2024-11-11T22:28:07,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/31298f08da8042eea8730cdd77b228fe is 50, key is test_row_0/C:col10/1731364084800/Put/seqid=0 2024-11-11T22:28:07,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742381_1557 (size=12151) 2024-11-11T22:28:07,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364147593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364147603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364147603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364147603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364147603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:07,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:07,826 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/31298f08da8042eea8730cdd77b228fe 2024-11-11T22:28:07,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/3e1a35b4ba1145018808a058f4d87c3c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/3e1a35b4ba1145018808a058f4d87c3c 2024-11-11T22:28:07,832 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/3e1a35b4ba1145018808a058f4d87c3c, entries=150, sequenceid=133, filesize=30.4 K 2024-11-11T22:28:07,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/8d3a92d3e32c43b59ae1d1940bfab03a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/8d3a92d3e32c43b59ae1d1940bfab03a 2024-11-11T22:28:07,835 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/8d3a92d3e32c43b59ae1d1940bfab03a, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T22:28:07,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/31298f08da8042eea8730cdd77b228fe as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/31298f08da8042eea8730cdd77b228fe 2024-11-11T22:28:07,839 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/31298f08da8042eea8730cdd77b228fe, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T22:28:07,840 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 86be86bb450634ab24cca4850cb6c7ec in 1741ms, sequenceid=133, compaction requested=false 2024-11-11T22:28:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-11T22:28:07,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-11T22:28:07,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-11T22:28:07,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1150 sec 2024-11-11T22:28:07,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 3.1200 sec 2024-11-11T22:28:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:08,107 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:28:08,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T22:28:08,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d55b5ee5feaf4735ad2575e15132b204_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364086967/Put/seqid=0 2024-11-11T22:28:08,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742382_1558 (size=14794) 2024-11-11T22:28:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364148119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364148120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364148121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364148122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364148123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364148233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364148233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364148234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364148234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364148234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364148442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364148443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364148443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364148443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364148443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,517 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:08,519 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d55b5ee5feaf4735ad2575e15132b204_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d55b5ee5feaf4735ad2575e15132b204_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:08,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/b3e2b822a2c646028e9b657cac72d6df, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:08,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/b3e2b822a2c646028e9b657cac72d6df is 175, key is test_row_0/A:col10/1731364086967/Put/seqid=0 2024-11-11T22:28:08,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742383_1559 (size=39749) 2024-11-11T22:28:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364148753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364148753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364148753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364148754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364148754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T22:28:08,893 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:28:08,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:28:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-11T22:28:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:28:08,895 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:28:08,896 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:28:08,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:28:08,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/b3e2b822a2c646028e9b657cac72d6df 2024-11-11T22:28:08,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/ab5e808958444ce38cef795d191919f4 is 50, key is test_row_0/B:col10/1731364086967/Put/seqid=0 2024-11-11T22:28:08,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742384_1560 (size=12151) 2024-11-11T22:28:08,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/ab5e808958444ce38cef795d191919f4 2024-11-11T22:28:08,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/c362f08ce2dd4bddbb6a5e0eda276cd5 is 50, key is test_row_0/C:col10/1731364086967/Put/seqid=0 2024-11-11T22:28:08,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742385_1561 (size=12151) 2024-11-11T22:28:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:28:09,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T22:28:09,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:09,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:09,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:09,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T22:28:09,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:09,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:09,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:28:09,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364149263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:09,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364149263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:09,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364149263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364149263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364149263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:09,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/c362f08ce2dd4bddbb6a5e0eda276cd5 2024-11-11T22:28:09,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/b3e2b822a2c646028e9b657cac72d6df as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b3e2b822a2c646028e9b657cac72d6df 2024-11-11T22:28:09,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b3e2b822a2c646028e9b657cac72d6df, entries=200, sequenceid=158, filesize=38.8 K 2024-11-11T22:28:09,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/ab5e808958444ce38cef795d191919f4 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/ab5e808958444ce38cef795d191919f4 2024-11-11T22:28:09,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T22:28:09,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:09,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:09,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:09,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/ab5e808958444ce38cef795d191919f4, entries=150, sequenceid=158, filesize=11.9 K 2024-11-11T22:28:09,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/c362f08ce2dd4bddbb6a5e0eda276cd5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/c362f08ce2dd4bddbb6a5e0eda276cd5 2024-11-11T22:28:09,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/c362f08ce2dd4bddbb6a5e0eda276cd5, entries=150, sequenceid=158, filesize=11.9 K 2024-11-11T22:28:09,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 86be86bb450634ab24cca4850cb6c7ec in 1253ms, sequenceid=158, compaction requested=true 2024-11-11T22:28:09,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:09,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:28:09,360 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:28:09,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:09,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:28:09,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:09,360 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:28:09,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:28:09,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:09,362 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:28:09,362 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/B is initiating minor compaction (all files) 2024-11-11T22:28:09,362 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/B in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,362 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a4a115d89e5947b38e8e979e3b0e4746, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/8d3a92d3e32c43b59ae1d1940bfab03a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/ab5e808958444ce38cef795d191919f4] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=35.7 K 2024-11-11T22:28:09,362 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:28:09,362 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/A is initiating minor compaction (all files) 2024-11-11T22:28:09,362 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/A in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,362 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/21a83c0bb7fa4e9eb48be13fbea89007, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/3e1a35b4ba1145018808a058f4d87c3c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b3e2b822a2c646028e9b657cac72d6df] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=99.7 K 2024-11-11T22:28:09,362 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,362 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/21a83c0bb7fa4e9eb48be13fbea89007, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/3e1a35b4ba1145018808a058f4d87c3c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b3e2b822a2c646028e9b657cac72d6df] 2024-11-11T22:28:09,363 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a115d89e5947b38e8e979e3b0e4746, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731364084464 2024-11-11T22:28:09,363 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21a83c0bb7fa4e9eb48be13fbea89007, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731364084464 2024-11-11T22:28:09,363 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d3a92d3e32c43b59ae1d1940bfab03a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731364084798 2024-11-11T22:28:09,363 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e1a35b4ba1145018808a058f4d87c3c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731364084798 2024-11-11T22:28:09,363 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5e808958444ce38cef795d191919f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731364086967 2024-11-11T22:28:09,363 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3e2b822a2c646028e9b657cac72d6df, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731364086967 2024-11-11T22:28:09,367 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:09,368 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:09,368 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/1a985271b7314784a7953f84f1774928 is 50, key is test_row_0/B:col10/1731364086967/Put/seqid=0 2024-11-11T22:28:09,369 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111300d07d6c9b4459eb1a99a9a8f81b7ff_86be86bb450634ab24cca4850cb6c7ec store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:09,371 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111300d07d6c9b4459eb1a99a9a8f81b7ff_86be86bb450634ab24cca4850cb6c7ec, store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:09,372 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111300d07d6c9b4459eb1a99a9a8f81b7ff_86be86bb450634ab24cca4850cb6c7ec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:09,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742386_1562 (size=12493) 2024-11-11T22:28:09,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742387_1563 (size=4469) 2024-11-11T22:28:09,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T22:28:09,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,506 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:28:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a707b0890b0741c18f49c03049483d20_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364088119/Put/seqid=0 2024-11-11T22:28:09,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742388_1564 (size=12304) 2024-11-11T22:28:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:28:09,782 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:09,783 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/a12e382b21aa4bf1a89b1ab5ad5e9810 is 175, key is test_row_0/A:col10/1731364086967/Put/seqid=0 2024-11-11T22:28:09,784 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/1a985271b7314784a7953f84f1774928 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/1a985271b7314784a7953f84f1774928 2024-11-11T22:28:09,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742389_1565 (size=31447) 2024-11-11T22:28:09,789 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/B of 86be86bb450634ab24cca4850cb6c7ec into 1a985271b7314784a7953f84f1774928(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:09,789 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:09,789 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/B, priority=13, startTime=1731364089360; duration=0sec 2024-11-11T22:28:09,789 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:09,789 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:B 2024-11-11T22:28:09,789 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:28:09,790 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:28:09,790 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/C is initiating minor compaction (all files) 2024-11-11T22:28:09,790 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/C in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:09,790 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/ae6b3e8527cf4df9a4b8466e68b06284, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/31298f08da8042eea8730cdd77b228fe, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/c362f08ce2dd4bddbb6a5e0eda276cd5] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=35.7 K 2024-11-11T22:28:09,790 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting ae6b3e8527cf4df9a4b8466e68b06284, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731364084464 2024-11-11T22:28:09,790 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 31298f08da8042eea8730cdd77b228fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731364084798 2024-11-11T22:28:09,790 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting c362f08ce2dd4bddbb6a5e0eda276cd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731364086967 2024-11-11T22:28:09,796 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#C#compaction#480 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:09,796 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/ae46844965e94f05a985356b668921e0 is 50, key is test_row_0/C:col10/1731364086967/Put/seqid=0 2024-11-11T22:28:09,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742390_1566 (size=12493) 2024-11-11T22:28:09,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:09,918 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a707b0890b0741c18f49c03049483d20_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a707b0890b0741c18f49c03049483d20_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:09,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:09,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3 is 175, key is test_row_0/A:col10/1731364088119/Put/seqid=0 2024-11-11T22:28:09,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742391_1567 (size=31105) 2024-11-11T22:28:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:28:10,189 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/a12e382b21aa4bf1a89b1ab5ad5e9810 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a12e382b21aa4bf1a89b1ab5ad5e9810 2024-11-11T22:28:10,193 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/A of 86be86bb450634ab24cca4850cb6c7ec into a12e382b21aa4bf1a89b1ab5ad5e9810(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:10,193 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:10,193 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/A, priority=13, startTime=1731364089360; duration=0sec 2024-11-11T22:28:10,193 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:10,193 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:A 2024-11-11T22:28:10,202 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/ae46844965e94f05a985356b668921e0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/ae46844965e94f05a985356b668921e0 2024-11-11T22:28:10,206 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/C of 86be86bb450634ab24cca4850cb6c7ec into ae46844965e94f05a985356b668921e0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:10,206 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:10,206 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/C, priority=13, startTime=1731364089360; duration=0sec 2024-11-11T22:28:10,206 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:10,206 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:C 2024-11-11T22:28:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:10,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364150295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364150297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364150298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364150299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364150299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,325 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3 2024-11-11T22:28:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/4f0aeed366bb4a2bbfb74cda92de0fcd is 50, key is test_row_0/B:col10/1731364088119/Put/seqid=0 2024-11-11T22:28:10,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742392_1568 (size=12151) 2024-11-11T22:28:10,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364150403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364150403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364150413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364150414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364150414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364150613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364150613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364150622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364150623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364150623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,733 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/4f0aeed366bb4a2bbfb74cda92de0fcd 2024-11-11T22:28:10,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/4be51360425c4223a280f1bf43143998 is 50, key is test_row_0/C:col10/1731364088119/Put/seqid=0 2024-11-11T22:28:10,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742393_1569 (size=12151) 2024-11-11T22:28:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364150923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364150923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364150933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364150933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364150933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:28:11,142 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/4be51360425c4223a280f1bf43143998 2024-11-11T22:28:11,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3 2024-11-11T22:28:11,150 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3, entries=150, sequenceid=172, filesize=30.4 K 2024-11-11T22:28:11,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/4f0aeed366bb4a2bbfb74cda92de0fcd as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4f0aeed366bb4a2bbfb74cda92de0fcd 2024-11-11T22:28:11,154 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4f0aeed366bb4a2bbfb74cda92de0fcd, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T22:28:11,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/4be51360425c4223a280f1bf43143998 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/4be51360425c4223a280f1bf43143998 2024-11-11T22:28:11,158 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/4be51360425c4223a280f1bf43143998, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T22:28:11,159 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 86be86bb450634ab24cca4850cb6c7ec in 1653ms, sequenceid=172, compaction requested=false 2024-11-11T22:28:11,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:11,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:11,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-11T22:28:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-11T22:28:11,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-11T22:28:11,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2650 sec 2024-11-11T22:28:11,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.2690 sec 2024-11-11T22:28:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:11,436 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:28:11,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T22:28:11,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b71738bc39574692ae4be0f4d1bec12a_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364090297/Put/seqid=0 2024-11-11T22:28:11,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742394_1570 (size=12304) 2024-11-11T22:28:11,458 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:11,462 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b71738bc39574692ae4be0f4d1bec12a_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b71738bc39574692ae4be0f4d1bec12a_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:11,463 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/5be3b6372cbd4575abd2b8ed1fa4d69a, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:11,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/5be3b6372cbd4575abd2b8ed1fa4d69a is 175, key is test_row_0/A:col10/1731364090297/Put/seqid=0 2024-11-11T22:28:11,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742395_1571 (size=31105) 2024-11-11T22:28:11,469 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/5be3b6372cbd4575abd2b8ed1fa4d69a 2024-11-11T22:28:11,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/5ddfceb98fe6453d94ae5ab272860f7b is 50, key is test_row_0/B:col10/1731364090297/Put/seqid=0 2024-11-11T22:28:11,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742396_1572 (size=12151) 2024-11-11T22:28:11,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364151491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364151496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364151498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364151505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364151505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364151614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364151614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364151614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364151623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364151623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364151823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364151823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364151823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364151833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364151833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:11,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/5ddfceb98fe6453d94ae5ab272860f7b 2024-11-11T22:28:11,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/72b664f6fd844aebaa65964575a3e830 is 50, key is test_row_0/C:col10/1731364090297/Put/seqid=0 2024-11-11T22:28:11,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742397_1573 (size=12151) 2024-11-11T22:28:12,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364152133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364152133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364152133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364152143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364152143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/72b664f6fd844aebaa65964575a3e830 2024-11-11T22:28:12,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/5be3b6372cbd4575abd2b8ed1fa4d69a as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5be3b6372cbd4575abd2b8ed1fa4d69a 2024-11-11T22:28:12,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5be3b6372cbd4575abd2b8ed1fa4d69a, entries=150, sequenceid=199, filesize=30.4 K 2024-11-11T22:28:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/5ddfceb98fe6453d94ae5ab272860f7b as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/5ddfceb98fe6453d94ae5ab272860f7b 2024-11-11T22:28:12,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/5ddfceb98fe6453d94ae5ab272860f7b, entries=150, sequenceid=199, filesize=11.9 K 2024-11-11T22:28:12,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/72b664f6fd844aebaa65964575a3e830 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/72b664f6fd844aebaa65964575a3e830 2024-11-11T22:28:12,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/72b664f6fd844aebaa65964575a3e830, entries=150, sequenceid=199, filesize=11.9 K 2024-11-11T22:28:12,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for 86be86bb450634ab24cca4850cb6c7ec in 877ms, sequenceid=199, compaction requested=true 2024-11-11T22:28:12,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:12,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:28:12,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:12,314 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:28:12,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:28:12,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:12,314 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:28:12,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:28:12,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:12,314 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:28:12,314 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:28:12,314 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/B is initiating minor compaction (all files) 2024-11-11T22:28:12,314 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/A is initiating minor compaction (all files) 2024-11-11T22:28:12,314 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/B in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:12,314 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/A in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:12,314 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a12e382b21aa4bf1a89b1ab5ad5e9810, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5be3b6372cbd4575abd2b8ed1fa4d69a] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=91.5 K 2024-11-11T22:28:12,314 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/1a985271b7314784a7953f84f1774928, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4f0aeed366bb4a2bbfb74cda92de0fcd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/5ddfceb98fe6453d94ae5ab272860f7b] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=35.9 K 2024-11-11T22:28:12,314 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:12,315 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a12e382b21aa4bf1a89b1ab5ad5e9810, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5be3b6372cbd4575abd2b8ed1fa4d69a] 2024-11-11T22:28:12,315 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting a12e382b21aa4bf1a89b1ab5ad5e9810, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731364086967 2024-11-11T22:28:12,315 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a985271b7314784a7953f84f1774928, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731364086967 2024-11-11T22:28:12,315 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fe5a9a6db2943f7ba0a6e23dcc35ae3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731364088119 2024-11-11T22:28:12,315 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f0aeed366bb4a2bbfb74cda92de0fcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731364088119 2024-11-11T22:28:12,316 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ddfceb98fe6453d94ae5ab272860f7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731364090293 2024-11-11T22:28:12,316 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5be3b6372cbd4575abd2b8ed1fa4d69a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731364090293 2024-11-11T22:28:12,319 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:12,320 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:12,320 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/d483791bed4744918faea4d8aa9d79e0 is 50, key is test_row_0/B:col10/1731364090297/Put/seqid=0 2024-11-11T22:28:12,320 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111104ca0c594cd44391976fb98507515ce0_86be86bb450634ab24cca4850cb6c7ec store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:12,322 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111104ca0c594cd44391976fb98507515ce0_86be86bb450634ab24cca4850cb6c7ec, store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:12,322 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111104ca0c594cd44391976fb98507515ce0_86be86bb450634ab24cca4850cb6c7ec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:12,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742398_1574 (size=12595) 2024-11-11T22:28:12,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742399_1575 (size=4469) 2024-11-11T22:28:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:12,645 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:28:12,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T22:28:12,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115f39c8384d224aab98fe025c50fe8935_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364092644/Put/seqid=0 2024-11-11T22:28:12,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742400_1576 (size=14794) 2024-11-11T22:28:12,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364152664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364152666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364152666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364152667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364152668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,727 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/d483791bed4744918faea4d8aa9d79e0 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/d483791bed4744918faea4d8aa9d79e0 2024-11-11T22:28:12,727 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:12,728 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/b21290b0569348b297f179a5a3eafdaf is 175, key is test_row_0/A:col10/1731364090297/Put/seqid=0 2024-11-11T22:28:12,731 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/B of 86be86bb450634ab24cca4850cb6c7ec into d483791bed4744918faea4d8aa9d79e0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:12,731 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:12,731 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/B, priority=13, startTime=1731364092314; duration=0sec 2024-11-11T22:28:12,731 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:12,731 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:B 2024-11-11T22:28:12,731 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:28:12,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742401_1577 (size=31549) 2024-11-11T22:28:12,732 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:28:12,732 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/C is initiating minor compaction (all files) 2024-11-11T22:28:12,732 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/C in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:12,732 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/ae46844965e94f05a985356b668921e0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/4be51360425c4223a280f1bf43143998, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/72b664f6fd844aebaa65964575a3e830] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=35.9 K 2024-11-11T22:28:12,732 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting ae46844965e94f05a985356b668921e0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731364086967 2024-11-11T22:28:12,733 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4be51360425c4223a280f1bf43143998, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731364088119 2024-11-11T22:28:12,733 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b664f6fd844aebaa65964575a3e830, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731364090293 2024-11-11T22:28:12,736 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/b21290b0569348b297f179a5a3eafdaf as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b21290b0569348b297f179a5a3eafdaf 2024-11-11T22:28:12,738 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:12,739 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/070c047298cd432884f6a1903f4a9290 is 50, key is test_row_0/C:col10/1731364090297/Put/seqid=0 2024-11-11T22:28:12,739 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/A of 86be86bb450634ab24cca4850cb6c7ec into b21290b0569348b297f179a5a3eafdaf(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:12,739 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:12,739 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/A, priority=13, startTime=1731364092313; duration=0sec 2024-11-11T22:28:12,739 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:12,739 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:A 2024-11-11T22:28:12,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742402_1578 (size=12595) 2024-11-11T22:28:12,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364152773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364152774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364152774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364152774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364152783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364152983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364152983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364152983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364152983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:12,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364152993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,054 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:13,057 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115f39c8384d224aab98fe025c50fe8935_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f39c8384d224aab98fe025c50fe8935_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:13,058 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/50b9ecc1796f4a08b63ea68c9b481a46, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:13,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/50b9ecc1796f4a08b63ea68c9b481a46 is 175, key is test_row_0/A:col10/1731364092644/Put/seqid=0 2024-11-11T22:28:13,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742403_1579 (size=39749) 2024-11-11T22:28:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:28:13,063 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:28:13,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:28:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-11T22:28:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:28:13,065 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:28:13,065 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:28:13,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:28:13,149 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/070c047298cd432884f6a1903f4a9290 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/070c047298cd432884f6a1903f4a9290 2024-11-11T22:28:13,152 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/C of 86be86bb450634ab24cca4850cb6c7ec into 070c047298cd432884f6a1903f4a9290(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:13,152 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:13,152 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/C, priority=13, startTime=1731364092314; duration=0sec 2024-11-11T22:28:13,152 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:13,152 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:C 2024-11-11T22:28:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:28:13,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T22:28:13,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:13,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364153293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:13,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:13,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364153293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:13,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364153293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:13,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364153293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364153303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T22:28:13,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:13,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:28:13,462 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/50b9ecc1796f4a08b63ea68c9b481a46 2024-11-11T22:28:13,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/a8f624d8f2674cf68b963120e98a28b9 is 50, key is test_row_0/B:col10/1731364092644/Put/seqid=0 2024-11-11T22:28:13,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742404_1580 (size=12151) 2024-11-11T22:28:13,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/a8f624d8f2674cf68b963120e98a28b9 2024-11-11T22:28:13,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/83ddf00aa133445483d032469e0b0697 is 50, key is test_row_0/C:col10/1731364092644/Put/seqid=0 2024-11-11T22:28:13,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742405_1581 (size=12151) 2024-11-11T22:28:13,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T22:28:13,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:13,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T22:28:13,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:13,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:28:13,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364153803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364153803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364153803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364153803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364153812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:13,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T22:28:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:13,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/83ddf00aa133445483d032469e0b0697 2024-11-11T22:28:13,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/50b9ecc1796f4a08b63ea68c9b481a46 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/50b9ecc1796f4a08b63ea68c9b481a46 2024-11-11T22:28:13,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/50b9ecc1796f4a08b63ea68c9b481a46, entries=200, sequenceid=214, filesize=38.8 K 2024-11-11T22:28:13,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/a8f624d8f2674cf68b963120e98a28b9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a8f624d8f2674cf68b963120e98a28b9 2024-11-11T22:28:13,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a8f624d8f2674cf68b963120e98a28b9, entries=150, sequenceid=214, filesize=11.9 K 2024-11-11T22:28:13,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/83ddf00aa133445483d032469e0b0697 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/83ddf00aa133445483d032469e0b0697 2024-11-11T22:28:13,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/83ddf00aa133445483d032469e0b0697, entries=150, sequenceid=214, filesize=11.9 K 2024-11-11T22:28:13,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 86be86bb450634ab24cca4850cb6c7ec in 1247ms, sequenceid=214, compaction requested=false 2024-11-11T22:28:13,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:13,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T22:28:13,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:13,980 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T22:28:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c033ce65d32f4c12a10279884fe4569c_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364092666/Put/seqid=0 2024-11-11T22:28:13,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742406_1582 (size=12304) 2024-11-11T22:28:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:28:14,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:14,391 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c033ce65d32f4c12a10279884fe4569c_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c033ce65d32f4c12a10279884fe4569c_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/43bf7bd22d6f477f954de3c33b5a5d46, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/43bf7bd22d6f477f954de3c33b5a5d46 is 175, key is test_row_0/A:col10/1731364092666/Put/seqid=0 2024-11-11T22:28:14,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742407_1583 (size=31105) 2024-11-11T22:28:14,395 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/43bf7bd22d6f477f954de3c33b5a5d46 2024-11-11T22:28:14,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/4c6b7a8ae3cf4754b0daa3a68abc244f is 50, key is test_row_0/B:col10/1731364092666/Put/seqid=0 2024-11-11T22:28:14,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742408_1584 (size=12151) 2024-11-11T22:28:14,404 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/4c6b7a8ae3cf4754b0daa3a68abc244f 2024-11-11T22:28:14,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/d68e323c8c204f02b8f6f3f121efb2da is 50, key is test_row_0/C:col10/1731364092666/Put/seqid=0 2024-11-11T22:28:14,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742409_1585 (size=12151) 2024-11-11T22:28:14,812 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/d68e323c8c204f02b8f6f3f121efb2da 2024-11-11T22:28:14,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/43bf7bd22d6f477f954de3c33b5a5d46 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/43bf7bd22d6f477f954de3c33b5a5d46 2024-11-11T22:28:14,819 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/43bf7bd22d6f477f954de3c33b5a5d46, entries=150, sequenceid=238, filesize=30.4 K 2024-11-11T22:28:14,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/4c6b7a8ae3cf4754b0daa3a68abc244f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4c6b7a8ae3cf4754b0daa3a68abc244f 2024-11-11T22:28:14,822 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4c6b7a8ae3cf4754b0daa3a68abc244f, entries=150, sequenceid=238, filesize=11.9 K 2024-11-11T22:28:14,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/d68e323c8c204f02b8f6f3f121efb2da as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/d68e323c8c204f02b8f6f3f121efb2da 2024-11-11T22:28:14,827 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/d68e323c8c204f02b8f6f3f121efb2da, entries=150, sequenceid=238, filesize=11.9 K 2024-11-11T22:28:14,828 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=60.38 KB/61830 for 86be86bb450634ab24cca4850cb6c7ec in 848ms, sequenceid=238, compaction requested=true 2024-11-11T22:28:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-11T22:28:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-11T22:28:14,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T22:28:14,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7640 sec 2024-11-11T22:28:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:14,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:28:14,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T22:28:14,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7680 sec 2024-11-11T22:28:14,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111158c640151a324078ab4397290ea43cce_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364094829/Put/seqid=0 2024-11-11T22:28:14,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742410_1586 (size=19774) 2024-11-11T22:28:14,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364154849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:14,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364154849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:14,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364154850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:14,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364154850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:14,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364154852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364154963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364154964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:14,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364154964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364154964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364154964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364155172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364155173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364155173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364155173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364155173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:28:15,214 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:28:15,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:28:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-11T22:28:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:28:15,216 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:28:15,216 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:28:15,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:28:15,240 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:15,243 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111158c640151a324078ab4397290ea43cce_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111158c640151a324078ab4397290ea43cce_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:15,243 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/9e2c26dd61ce4cb0ad3abe601fc1eafc, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:15,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/9e2c26dd61ce4cb0ad3abe601fc1eafc is 175, key is test_row_0/A:col10/1731364094829/Put/seqid=0 2024-11-11T22:28:15,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742411_1587 (size=57033) 2024-11-11T22:28:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:28:15,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:28:15,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:15,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364155483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364155483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364155484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364155484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364155484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:28:15,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:15,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:15,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:15,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:28:15,647 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/9e2c26dd61ce4cb0ad3abe601fc1eafc 2024-11-11T22:28:15,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/bc13ec05d78d4f80a66c2018be924f6f is 50, key is test_row_0/B:col10/1731364094829/Put/seqid=0 2024-11-11T22:28:15,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742412_1588 (size=12151) 2024-11-11T22:28:15,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:28:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:15,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:28:15,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:15,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:15,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:15,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:28:15,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:28:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:15,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:15,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364155993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364155993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364155993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364155993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364155993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:16,033 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:28:16,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/bc13ec05d78d4f80a66c2018be924f6f 2024-11-11T22:28:16,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/fac1bf00a2824c5681ed2b9f8cc71d7d is 50, key is test_row_0/C:col10/1731364094829/Put/seqid=0 2024-11-11T22:28:16,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742413_1589 (size=12151) 2024-11-11T22:28:16,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:28:16,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:16,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:16,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:16,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:28:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:28:16,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:28:16,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:28:16,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/fac1bf00a2824c5681ed2b9f8cc71d7d 2024-11-11T22:28:16,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/9e2c26dd61ce4cb0ad3abe601fc1eafc as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/9e2c26dd61ce4cb0ad3abe601fc1eafc 2024-11-11T22:28:16,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/9e2c26dd61ce4cb0ad3abe601fc1eafc, entries=300, sequenceid=253, filesize=55.7 K 2024-11-11T22:28:16,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/bc13ec05d78d4f80a66c2018be924f6f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/bc13ec05d78d4f80a66c2018be924f6f 2024-11-11T22:28:16,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/bc13ec05d78d4f80a66c2018be924f6f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-11T22:28:16,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/fac1bf00a2824c5681ed2b9f8cc71d7d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/fac1bf00a2824c5681ed2b9f8cc71d7d 2024-11-11T22:28:16,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/fac1bf00a2824c5681ed2b9f8cc71d7d, entries=150, sequenceid=253, filesize=11.9 K 2024-11-11T22:28:16,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 86be86bb450634ab24cca4850cb6c7ec in 1648ms, sequenceid=253, compaction requested=true 2024-11-11T22:28:16,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:28:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:16,479 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:28:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:28:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:28:16,479 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:28:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:16,479 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:28:16,479 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159436 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:28:16,480 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/B is initiating minor compaction (all files) 2024-11-11T22:28:16,480 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/A is initiating minor compaction (all files) 2024-11-11T22:28:16,480 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/B in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,480 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/A in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,480 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b21290b0569348b297f179a5a3eafdaf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/50b9ecc1796f4a08b63ea68c9b481a46, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/43bf7bd22d6f477f954de3c33b5a5d46, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/9e2c26dd61ce4cb0ad3abe601fc1eafc] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=155.7 K 2024-11-11T22:28:16,480 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/d483791bed4744918faea4d8aa9d79e0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a8f624d8f2674cf68b963120e98a28b9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4c6b7a8ae3cf4754b0daa3a68abc244f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/bc13ec05d78d4f80a66c2018be924f6f] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=47.9 K 2024-11-11T22:28:16,480 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,480 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. files: [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b21290b0569348b297f179a5a3eafdaf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/50b9ecc1796f4a08b63ea68c9b481a46, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/43bf7bd22d6f477f954de3c33b5a5d46, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/9e2c26dd61ce4cb0ad3abe601fc1eafc] 2024-11-11T22:28:16,480 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting d483791bed4744918faea4d8aa9d79e0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731364090293 2024-11-11T22:28:16,480 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting b21290b0569348b297f179a5a3eafdaf, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731364090293 2024-11-11T22:28:16,480 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting a8f624d8f2674cf68b963120e98a28b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731364091490 2024-11-11T22:28:16,480 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50b9ecc1796f4a08b63ea68c9b481a46, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731364091490 2024-11-11T22:28:16,481 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c6b7a8ae3cf4754b0daa3a68abc244f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731364092664 2024-11-11T22:28:16,481 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43bf7bd22d6f477f954de3c33b5a5d46, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731364092664 2024-11-11T22:28:16,481 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting bc13ec05d78d4f80a66c2018be924f6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731364094829 2024-11-11T22:28:16,481 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e2c26dd61ce4cb0ad3abe601fc1eafc, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731364094824 2024-11-11T22:28:16,486 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:16,488 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#B#compaction#499 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:16,488 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/f183a4736b90407681dc825dce4701f9 is 50, key is test_row_0/B:col10/1731364094829/Put/seqid=0 2024-11-11T22:28:16,490 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411111988e52a0fe04c728f722d93f932d845_86be86bb450634ab24cca4850cb6c7ec store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:16,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742414_1590 (size=12731) 2024-11-11T22:28:16,492 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411111988e52a0fe04c728f722d93f932d845_86be86bb450634ab24cca4850cb6c7ec, store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:16,492 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111988e52a0fe04c728f722d93f932d845_86be86bb450634ab24cca4850cb6c7ec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:16,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742415_1591 (size=4469) 2024-11-11T22:28:16,501 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/f183a4736b90407681dc825dce4701f9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/f183a4736b90407681dc825dce4701f9 2024-11-11T22:28:16,503 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#A#compaction#498 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:16,503 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/11009541664b47fdb269a2331b44c19d is 175, key is test_row_0/A:col10/1731364094829/Put/seqid=0 2024-11-11T22:28:16,507 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/B of 86be86bb450634ab24cca4850cb6c7ec into f183a4736b90407681dc825dce4701f9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:16,507 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:16,507 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/B, priority=12, startTime=1731364096479; duration=0sec 2024-11-11T22:28:16,507 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:16,507 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:B 2024-11-11T22:28:16,507 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:28:16,508 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:28:16,508 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1541): 86be86bb450634ab24cca4850cb6c7ec/C is initiating minor compaction (all files) 2024-11-11T22:28:16,508 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86be86bb450634ab24cca4850cb6c7ec/C in TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,508 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/070c047298cd432884f6a1903f4a9290, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/83ddf00aa133445483d032469e0b0697, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/d68e323c8c204f02b8f6f3f121efb2da, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/fac1bf00a2824c5681ed2b9f8cc71d7d] into tmpdir=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp, totalSize=47.9 K 2024-11-11T22:28:16,509 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 070c047298cd432884f6a1903f4a9290, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731364090293 2024-11-11T22:28:16,509 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting 83ddf00aa133445483d032469e0b0697, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731364091490 2024-11-11T22:28:16,509 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting d68e323c8c204f02b8f6f3f121efb2da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731364092664 2024-11-11T22:28:16,510 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] compactions.Compactor(225): Compacting fac1bf00a2824c5681ed2b9f8cc71d7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731364094829 2024-11-11T22:28:16,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742416_1592 (size=31685) 2024-11-11T22:28:16,516 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86be86bb450634ab24cca4850cb6c7ec#C#compaction#500 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:28:16,516 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/2c3ad8143a184bc5926f149b6db0dbe7 is 50, key is test_row_0/C:col10/1731364094829/Put/seqid=0 2024-11-11T22:28:16,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742417_1593 (size=12731) 2024-11-11T22:28:16,525 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/2c3ad8143a184bc5926f149b6db0dbe7 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/2c3ad8143a184bc5926f149b6db0dbe7 2024-11-11T22:28:16,528 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/C of 86be86bb450634ab24cca4850cb6c7ec into 2c3ad8143a184bc5926f149b6db0dbe7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:16,528 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:16,529 INFO [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/C, priority=12, startTime=1731364096479; duration=0sec 2024-11-11T22:28:16,529 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:16,529 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:C 2024-11-11T22:28:16,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39613 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:28:16,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:16,589 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T22:28:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ddf0ba7fe7a04430a685c6cc75ad226b_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364094851/Put/seqid=0 2024-11-11T22:28:16,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742418_1594 (size=12454) 2024-11-11T22:28:16,914 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/11009541664b47fdb269a2331b44c19d as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/11009541664b47fdb269a2331b44c19d 2024-11-11T22:28:16,917 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86be86bb450634ab24cca4850cb6c7ec/A of 86be86bb450634ab24cca4850cb6c7ec into 11009541664b47fdb269a2331b44c19d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:28:16,917 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:16,917 INFO [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., storeName=86be86bb450634ab24cca4850cb6c7ec/A, priority=12, startTime=1731364096479; duration=0sec 2024-11-11T22:28:16,917 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:16,917 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:A 2024-11-11T22:28:16,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:17,000 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ddf0ba7fe7a04430a685c6cc75ad226b_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ddf0ba7fe7a04430a685c6cc75ad226b_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:17,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/0a27c4afc21d4ceabb381895de56f3f2, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:17,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/0a27c4afc21d4ceabb381895de56f3f2 is 175, key is test_row_0/A:col10/1731364094851/Put/seqid=0 2024-11-11T22:28:17,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742419_1595 (size=31255) 2024-11-11T22:28:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:17,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. as already flushing 2024-11-11T22:28:17,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364157014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364157015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364157017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364157017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364157017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364157123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364157123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364157123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364157123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364157133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,273 INFO [Thread-2342 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2342. 2024-11-11T22:28:17,273 DEBUG [Thread-2342 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:28:17,273 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:17,273 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:17,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:17,274 INFO [Thread-2346 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2346. 2024-11-11T22:28:17,274 DEBUG [Thread-2346 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:28:17,274 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:17,274 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:17,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:17,274 INFO [Thread-2348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2348. 2024-11-11T22:28:17,274 DEBUG [Thread-2348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:28:17,274 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:17,274 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:17,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:17,276 INFO [Thread-2344 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2344. 2024-11-11T22:28:17,277 DEBUG [Thread-2344 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:28:17,277 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:17,277 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:17,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:17,277 INFO [Thread-2340 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2340. 2024-11-11T22:28:17,277 DEBUG [Thread-2340 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:28:17,277 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:17,277 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:17,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:17,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364157333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364157333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364157334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364157334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364157343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:28:17,406 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/0a27c4afc21d4ceabb381895de56f3f2 2024-11-11T22:28:17,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/d5f888a767894a8fb5cd03cb1974c3ba is 50, key is test_row_0/B:col10/1731364094851/Put/seqid=0 2024-11-11T22:28:17,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742420_1596 (size=12301) 2024-11-11T22:28:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364157643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364157644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364157643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364157644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364157653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:17,888 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/d5f888a767894a8fb5cd03cb1974c3ba 2024-11-11T22:28:17,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/675fef4253ab4e0eb15e25b9f52f1ff9 is 50, key is test_row_0/C:col10/1731364094851/Put/seqid=0 2024-11-11T22:28:17,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742421_1597 (size=12301) 2024-11-11T22:28:18,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60378 deadline: 1731364158153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:18,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:18,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60338 deadline: 1731364158154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:18,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60330 deadline: 1731364158154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60350 deadline: 1731364158154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:18,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:18,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:28:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1731364158163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 , the old value is region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:28:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., hostname=8d82b89dd27d,39613,1731363918313, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:28:18,299 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/675fef4253ab4e0eb15e25b9f52f1ff9 2024-11-11T22:28:18,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/0a27c4afc21d4ceabb381895de56f3f2 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/0a27c4afc21d4ceabb381895de56f3f2 2024-11-11T22:28:18,311 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/0a27c4afc21d4ceabb381895de56f3f2, entries=150, sequenceid=276, filesize=30.5 K 2024-11-11T22:28:18,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/d5f888a767894a8fb5cd03cb1974c3ba as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/d5f888a767894a8fb5cd03cb1974c3ba 2024-11-11T22:28:18,317 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/d5f888a767894a8fb5cd03cb1974c3ba, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T22:28:18,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/675fef4253ab4e0eb15e25b9f52f1ff9 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/675fef4253ab4e0eb15e25b9f52f1ff9 2024-11-11T22:28:18,322 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/675fef4253ab4e0eb15e25b9f52f1ff9, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T22:28:18,323 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 86be86bb450634ab24cca4850cb6c7ec in 1733ms, sequenceid=276, compaction requested=false 2024-11-11T22:28:18,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:18,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:18,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-11T22:28:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-11T22:28:18,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T22:28:18,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1080 sec 2024-11-11T22:28:18,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.1120 sec 2024-11-11T22:28:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39613 {}] regionserver.HRegion(8855): Flush requested on 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:19,169 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T22:28:19,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-11T22:28:19,170 INFO [Thread-2337 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2337. 2024-11-11T22:28:19,170 INFO [Thread-2335 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2335. 2024-11-11T22:28:19,170 DEBUG [Thread-2337 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:28:19,170 INFO [Thread-2329 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2329. 2024-11-11T22:28:19,170 DEBUG [Thread-2335 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:28:19,170 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,170 DEBUG [Thread-2329 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:28:19,170 DEBUG [Thread-2335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,170 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,170 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,170 DEBUG [Thread-2335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,170 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:19,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:19,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:19,175 INFO [Thread-2331 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2331. 2024-11-11T22:28:19,175 INFO [Thread-2333 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2333. 2024-11-11T22:28:19,175 DEBUG [Thread-2331 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:28:19,175 DEBUG [Thread-2333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:28:19,175 DEBUG [Thread-2331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,175 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,175 DEBUG [Thread-2331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,175 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:19,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:19,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111162adae0061c0481e9eb6fabb891680e9_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364097016/Put/seqid=0 2024-11-11T22:28:19,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742422_1598 (size=12454) 2024-11-11T22:28:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:28:19,374 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4074 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4111 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4037 2024-11-11T22:28:19,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4095 2024-11-11T22:28:19,375 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4101 2024-11-11T22:28:19,375 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:28:19,375 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:28:19,375 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:28:19,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:19,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:19,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:28:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:28:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:28:19,385 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364099385"}]},"ts":"1731364099385"} 2024-11-11T22:28:19,387 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:28:19,387 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:28:19,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:28:19,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, UNASSIGN}] 2024-11-11T22:28:19,390 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, UNASSIGN 2024-11-11T22:28:19,391 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=86be86bb450634ab24cca4850cb6c7ec, regionState=CLOSING, regionLocation=8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:19,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, UNASSIGN because future has completed 2024-11-11T22:28:19,392 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:28:19,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313}] 2024-11-11T22:28:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:28:19,545 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(122): Close 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:19,545 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:28:19,545 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1722): Closing 86be86bb450634ab24cca4850cb6c7ec, disabling compactions & flushes 2024-11-11T22:28:19,546 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:19,582 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:19,592 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111162adae0061c0481e9eb6fabb891680e9_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111162adae0061c0481e9eb6fabb891680e9_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:19,593 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/47bd27ccaf48418db0b748bbaec8f550, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:19,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/47bd27ccaf48418db0b748bbaec8f550 is 175, key is test_row_0/A:col10/1731364097016/Put/seqid=0 2024-11-11T22:28:19,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742423_1599 (size=31255) 2024-11-11T22:28:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:28:19,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/47bd27ccaf48418db0b748bbaec8f550 2024-11-11T22:28:20,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/f01574a66aec409c9259b925ce470b05 is 50, key is test_row_0/B:col10/1731364097016/Put/seqid=0 2024-11-11T22:28:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:28:20,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742424_1600 (size=12301) 2024-11-11T22:28:20,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/f01574a66aec409c9259b925ce470b05 2024-11-11T22:28:20,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/9c825445a57e4d90b5e1e724d4a98135 is 50, key is test_row_0/C:col10/1731364097016/Put/seqid=0 2024-11-11T22:28:20,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742425_1601 (size=12301) 2024-11-11T22:28:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:28:20,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/9c825445a57e4d90b5e1e724d4a98135 2024-11-11T22:28:20,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/47bd27ccaf48418db0b748bbaec8f550 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/47bd27ccaf48418db0b748bbaec8f550 2024-11-11T22:28:20,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/47bd27ccaf48418db0b748bbaec8f550, entries=150, sequenceid=295, filesize=30.5 K 2024-11-11T22:28:20,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/f01574a66aec409c9259b925ce470b05 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/f01574a66aec409c9259b925ce470b05 2024-11-11T22:28:20,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/f01574a66aec409c9259b925ce470b05, entries=150, sequenceid=295, filesize=12.0 K 2024-11-11T22:28:20,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/9c825445a57e4d90b5e1e724d4a98135 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/9c825445a57e4d90b5e1e724d4a98135 2024-11-11T22:28:20,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/9c825445a57e4d90b5e1e724d4a98135, entries=150, sequenceid=295, filesize=12.0 K 2024-11-11T22:28:20,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=13.42 KB/13740 for 86be86bb450634ab24cca4850cb6c7ec in 1687ms, sequenceid=295, compaction requested=true 2024-11-11T22:28:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86be86bb450634ab24cca4850cb6c7ec: 2024-11-11T22:28:20,856 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:28:20,856 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:20,856 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. after waiting 0 ms 2024-11-11T22:28:20,856 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:28:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:28:20,856 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(2902): Flushing 86be86bb450634ab24cca4850cb6c7ec 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-11T22:28:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86be86bb450634ab24cca4850cb6c7ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:28:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:28:20,856 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. because compaction request was cancelled 2024-11-11T22:28:20,856 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:A 2024-11-11T22:28:20,856 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. because compaction request was cancelled 2024-11-11T22:28:20,856 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. because compaction request was cancelled 2024-11-11T22:28:20,856 DEBUG [RS:0;8d82b89dd27d:39613-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:B 2024-11-11T22:28:20,856 DEBUG [RS:0;8d82b89dd27d:39613-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86be86bb450634ab24cca4850cb6c7ec:C 2024-11-11T22:28:20,860 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115680dd60e7194fbc941cd7224cb3f0bc_86be86bb450634ab24cca4850cb6c7ec is 50, key is test_row_0/A:col10/1731364099174/Put/seqid=0 2024-11-11T22:28:20,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742426_1602 (size=9914) 2024-11-11T22:28:21,264 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:28:21,272 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115680dd60e7194fbc941cd7224cb3f0bc_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115680dd60e7194fbc941cd7224cb3f0bc_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:21,274 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/b58e0e51a9d648989220fef086fd441f, store: [table=TestAcidGuarantees family=A region=86be86bb450634ab24cca4850cb6c7ec] 2024-11-11T22:28:21,275 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/b58e0e51a9d648989220fef086fd441f is 175, key is test_row_0/A:col10/1731364099174/Put/seqid=0 2024-11-11T22:28:21,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742427_1603 (size=22561) 2024-11-11T22:28:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:28:21,679 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/b58e0e51a9d648989220fef086fd441f 2024-11-11T22:28:21,684 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/1e63720cf98e473dade0803c31183ad5 is 50, key is test_row_0/B:col10/1731364099174/Put/seqid=0 2024-11-11T22:28:21,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742428_1604 (size=9857) 2024-11-11T22:28:22,087 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/1e63720cf98e473dade0803c31183ad5 2024-11-11T22:28:22,093 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/323c06acc5e7446694cc9c241a1e494c is 50, key is test_row_0/C:col10/1731364099174/Put/seqid=0 2024-11-11T22:28:22,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742429_1605 (size=9857) 2024-11-11T22:28:22,498 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/323c06acc5e7446694cc9c241a1e494c 2024-11-11T22:28:22,501 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/A/b58e0e51a9d648989220fef086fd441f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b58e0e51a9d648989220fef086fd441f 2024-11-11T22:28:22,505 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b58e0e51a9d648989220fef086fd441f, entries=100, sequenceid=300, filesize=22.0 K 2024-11-11T22:28:22,505 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/B/1e63720cf98e473dade0803c31183ad5 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/1e63720cf98e473dade0803c31183ad5 2024-11-11T22:28:22,509 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/1e63720cf98e473dade0803c31183ad5, entries=100, sequenceid=300, filesize=9.6 K 2024-11-11T22:28:22,509 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/.tmp/C/323c06acc5e7446694cc9c241a1e494c as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/323c06acc5e7446694cc9c241a1e494c 2024-11-11T22:28:22,512 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/323c06acc5e7446694cc9c241a1e494c, entries=100, sequenceid=300, filesize=9.6 K 2024-11-11T22:28:22,513 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 86be86bb450634ab24cca4850cb6c7ec in 1657ms, sequenceid=300, compaction requested=true 2024-11-11T22:28:22,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/1cfdc166559a45c4af95d2649fe69f13, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/2849b6e0d2cc4525bca5350bcc5d486f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/c4a4c6b89b1b4662a378ff9d023387f4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/334508f76c0748828a914b5006a3776c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/99637cc4fb394a7fa2ad2f8ea151c4da, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a070faab75984a8696dbb6330cc9a32c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/466154ba5dd6401e957c9c3535985419, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/21a83c0bb7fa4e9eb48be13fbea89007, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/3e1a35b4ba1145018808a058f4d87c3c, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b3e2b822a2c646028e9b657cac72d6df, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a12e382b21aa4bf1a89b1ab5ad5e9810, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b21290b0569348b297f179a5a3eafdaf, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5be3b6372cbd4575abd2b8ed1fa4d69a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/50b9ecc1796f4a08b63ea68c9b481a46, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/43bf7bd22d6f477f954de3c33b5a5d46, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/9e2c26dd61ce4cb0ad3abe601fc1eafc] to archive 2024-11-11T22:28:22,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:28:22,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/1cfdc166559a45c4af95d2649fe69f13 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/1cfdc166559a45c4af95d2649fe69f13 2024-11-11T22:28:22,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/2849b6e0d2cc4525bca5350bcc5d486f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/2849b6e0d2cc4525bca5350bcc5d486f 2024-11-11T22:28:22,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/c4a4c6b89b1b4662a378ff9d023387f4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/c4a4c6b89b1b4662a378ff9d023387f4 2024-11-11T22:28:22,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/334508f76c0748828a914b5006a3776c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/334508f76c0748828a914b5006a3776c 2024-11-11T22:28:22,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/99637cc4fb394a7fa2ad2f8ea151c4da to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/99637cc4fb394a7fa2ad2f8ea151c4da 2024-11-11T22:28:22,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a070faab75984a8696dbb6330cc9a32c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a070faab75984a8696dbb6330cc9a32c 2024-11-11T22:28:22,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/466154ba5dd6401e957c9c3535985419 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/466154ba5dd6401e957c9c3535985419 2024-11-11T22:28:22,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/21a83c0bb7fa4e9eb48be13fbea89007 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/21a83c0bb7fa4e9eb48be13fbea89007 2024-11-11T22:28:22,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/3e1a35b4ba1145018808a058f4d87c3c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/3e1a35b4ba1145018808a058f4d87c3c 2024-11-11T22:28:22,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b3e2b822a2c646028e9b657cac72d6df to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b3e2b822a2c646028e9b657cac72d6df 2024-11-11T22:28:22,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a12e382b21aa4bf1a89b1ab5ad5e9810 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/a12e382b21aa4bf1a89b1ab5ad5e9810 2024-11-11T22:28:22,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5fe5a9a6db2943f7ba0a6e23dcc35ae3 2024-11-11T22:28:22,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b21290b0569348b297f179a5a3eafdaf to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b21290b0569348b297f179a5a3eafdaf 2024-11-11T22:28:22,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5be3b6372cbd4575abd2b8ed1fa4d69a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/5be3b6372cbd4575abd2b8ed1fa4d69a 2024-11-11T22:28:22,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/50b9ecc1796f4a08b63ea68c9b481a46 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/50b9ecc1796f4a08b63ea68c9b481a46 2024-11-11T22:28:22,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/43bf7bd22d6f477f954de3c33b5a5d46 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/43bf7bd22d6f477f954de3c33b5a5d46 2024-11-11T22:28:22,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/9e2c26dd61ce4cb0ad3abe601fc1eafc to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/9e2c26dd61ce4cb0ad3abe601fc1eafc 2024-11-11T22:28:22,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/fec234d0c94c403dbf5980bce25b2559, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/da4efa9880974ee9bd6ed40cd1dc29fe, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2958804b414248868260cc23637ea097, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/360c0d13fe88421e8f12f12406498951, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/69a055111bca41fd85eb35df5be9add4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/e45e309a7b144612af8368fce2708346, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a4a115d89e5947b38e8e979e3b0e4746, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2dbb6711fbd242708477d42a374d6669, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/8d3a92d3e32c43b59ae1d1940bfab03a, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/1a985271b7314784a7953f84f1774928, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/ab5e808958444ce38cef795d191919f4, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4f0aeed366bb4a2bbfb74cda92de0fcd, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/d483791bed4744918faea4d8aa9d79e0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/5ddfceb98fe6453d94ae5ab272860f7b, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a8f624d8f2674cf68b963120e98a28b9, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4c6b7a8ae3cf4754b0daa3a68abc244f, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/bc13ec05d78d4f80a66c2018be924f6f] to archive 2024-11-11T22:28:22,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:28:22,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/fec234d0c94c403dbf5980bce25b2559 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/fec234d0c94c403dbf5980bce25b2559 2024-11-11T22:28:22,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/da4efa9880974ee9bd6ed40cd1dc29fe to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/da4efa9880974ee9bd6ed40cd1dc29fe 2024-11-11T22:28:22,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2958804b414248868260cc23637ea097 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2958804b414248868260cc23637ea097 2024-11-11T22:28:22,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/360c0d13fe88421e8f12f12406498951 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/360c0d13fe88421e8f12f12406498951 2024-11-11T22:28:22,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/69a055111bca41fd85eb35df5be9add4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/69a055111bca41fd85eb35df5be9add4 2024-11-11T22:28:22,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/e45e309a7b144612af8368fce2708346 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/e45e309a7b144612af8368fce2708346 2024-11-11T22:28:22,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a4a115d89e5947b38e8e979e3b0e4746 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a4a115d89e5947b38e8e979e3b0e4746 2024-11-11T22:28:22,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2dbb6711fbd242708477d42a374d6669 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/2dbb6711fbd242708477d42a374d6669 2024-11-11T22:28:22,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/8d3a92d3e32c43b59ae1d1940bfab03a to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/8d3a92d3e32c43b59ae1d1940bfab03a 2024-11-11T22:28:22,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/1a985271b7314784a7953f84f1774928 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/1a985271b7314784a7953f84f1774928 2024-11-11T22:28:22,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/ab5e808958444ce38cef795d191919f4 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/ab5e808958444ce38cef795d191919f4 2024-11-11T22:28:22,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4f0aeed366bb4a2bbfb74cda92de0fcd to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4f0aeed366bb4a2bbfb74cda92de0fcd 2024-11-11T22:28:22,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/d483791bed4744918faea4d8aa9d79e0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/d483791bed4744918faea4d8aa9d79e0 2024-11-11T22:28:22,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/5ddfceb98fe6453d94ae5ab272860f7b to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/5ddfceb98fe6453d94ae5ab272860f7b 2024-11-11T22:28:22,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a8f624d8f2674cf68b963120e98a28b9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/a8f624d8f2674cf68b963120e98a28b9 2024-11-11T22:28:22,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4c6b7a8ae3cf4754b0daa3a68abc244f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/4c6b7a8ae3cf4754b0daa3a68abc244f 2024-11-11T22:28:22,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/bc13ec05d78d4f80a66c2018be924f6f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/bc13ec05d78d4f80a66c2018be924f6f 2024-11-11T22:28:22,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/38815b8d0f73439b98df2e066e1279e3, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/dfa229024ef0439db91448cd0aec56fb, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a5c5618b4f784b97aff932c2477daf6d, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/f108bd8c470c42d6813412fe6ed55c86, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a46b2ca1594848b184de14f583138e01, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/444ec7f5ba294d27a951715271a77b25, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/ae6b3e8527cf4df9a4b8466e68b06284, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/15fe41f4e8a04a65af3d7ba890d1ddc1, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/31298f08da8042eea8730cdd77b228fe, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/ae46844965e94f05a985356b668921e0, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/c362f08ce2dd4bddbb6a5e0eda276cd5, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/4be51360425c4223a280f1bf43143998, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/070c047298cd432884f6a1903f4a9290, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/72b664f6fd844aebaa65964575a3e830, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/83ddf00aa133445483d032469e0b0697, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/d68e323c8c204f02b8f6f3f121efb2da, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/fac1bf00a2824c5681ed2b9f8cc71d7d] to archive 2024-11-11T22:28:22,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:28:22,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/38815b8d0f73439b98df2e066e1279e3 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/38815b8d0f73439b98df2e066e1279e3 2024-11-11T22:28:22,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/dfa229024ef0439db91448cd0aec56fb to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/dfa229024ef0439db91448cd0aec56fb 2024-11-11T22:28:22,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a5c5618b4f784b97aff932c2477daf6d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a5c5618b4f784b97aff932c2477daf6d 2024-11-11T22:28:22,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/f108bd8c470c42d6813412fe6ed55c86 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/f108bd8c470c42d6813412fe6ed55c86 2024-11-11T22:28:22,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a46b2ca1594848b184de14f583138e01 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/a46b2ca1594848b184de14f583138e01 2024-11-11T22:28:22,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/444ec7f5ba294d27a951715271a77b25 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/444ec7f5ba294d27a951715271a77b25 2024-11-11T22:28:22,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/ae6b3e8527cf4df9a4b8466e68b06284 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/ae6b3e8527cf4df9a4b8466e68b06284 2024-11-11T22:28:22,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/15fe41f4e8a04a65af3d7ba890d1ddc1 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/15fe41f4e8a04a65af3d7ba890d1ddc1 2024-11-11T22:28:22,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/31298f08da8042eea8730cdd77b228fe to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/31298f08da8042eea8730cdd77b228fe 2024-11-11T22:28:22,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/ae46844965e94f05a985356b668921e0 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/ae46844965e94f05a985356b668921e0 2024-11-11T22:28:22,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/c362f08ce2dd4bddbb6a5e0eda276cd5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/c362f08ce2dd4bddbb6a5e0eda276cd5 2024-11-11T22:28:22,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/4be51360425c4223a280f1bf43143998 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/4be51360425c4223a280f1bf43143998 2024-11-11T22:28:22,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/070c047298cd432884f6a1903f4a9290 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/070c047298cd432884f6a1903f4a9290 2024-11-11T22:28:22,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/72b664f6fd844aebaa65964575a3e830 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/72b664f6fd844aebaa65964575a3e830 2024-11-11T22:28:22,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/83ddf00aa133445483d032469e0b0697 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/83ddf00aa133445483d032469e0b0697 2024-11-11T22:28:22,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/d68e323c8c204f02b8f6f3f121efb2da to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/d68e323c8c204f02b8f6f3f121efb2da 2024-11-11T22:28:22,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/fac1bf00a2824c5681ed2b9f8cc71d7d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/fac1bf00a2824c5681ed2b9f8cc71d7d 2024-11-11T22:28:22,576 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/recovered.edits/303.seqid, newMaxSeqId=303, maxSeqId=4 2024-11-11T22:28:22,576 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. 2024-11-11T22:28:22,576 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1676): Region close journal for 86be86bb450634ab24cca4850cb6c7ec: Waiting for close lock at 1731364099545Running coprocessor pre-close hooks at 1731364099545Disabling compacts and flushes for region at 1731364099545Disabling writes for close at 1731364100856 (+1311 ms)Obtaining lock to block concurrent updates at 1731364100856Preparing flush snapshotting stores in 86be86bb450634ab24cca4850cb6c7ec at 1731364100856Finished memstore snapshotting TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731364100856Flushing stores of TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec. at 1731364100857 (+1 ms)Flushing 86be86bb450634ab24cca4850cb6c7ec/A: creating writer at 1731364100857Flushing mob file 86be86bb450634ab24cca4850cb6c7ec/A: appending metadata at 1731364100860 (+3 ms)Flushing mob file 86be86bb450634ab24cca4850cb6c7ec/A: closing flushed file at 1731364100860Flushing 86be86bb450634ab24cca4850cb6c7ec/A: appending metadata at 1731364101274 (+414 ms)Flushing 86be86bb450634ab24cca4850cb6c7ec/A: closing flushed file at 1731364101274Flushing 86be86bb450634ab24cca4850cb6c7ec/B: creating writer at 1731364101682 (+408 ms)Flushing 86be86bb450634ab24cca4850cb6c7ec/B: appending metadata at 1731364101684 (+2 ms)Flushing 86be86bb450634ab24cca4850cb6c7ec/B: closing flushed file at 1731364101684Flushing 86be86bb450634ab24cca4850cb6c7ec/C: creating writer at 1731364102090 (+406 ms)Flushing 86be86bb450634ab24cca4850cb6c7ec/C: appending metadata at 1731364102093 (+3 ms)Flushing 86be86bb450634ab24cca4850cb6c7ec/C: closing flushed file at 1731364102093Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43acb1b0: reopening flushed file at 1731364102501 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49468fe4: reopening flushed file at 1731364102505 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78c5e8bd: reopening flushed file at 1731364102509 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 86be86bb450634ab24cca4850cb6c7ec in 1657ms, sequenceid=300, compaction requested=true at 1731364102513 (+4 ms)Writing region close event to WAL at 1731364102573 (+60 ms)Running coprocessor post-close hooks at 1731364102576 (+3 ms)Closed at 1731364102576 2024-11-11T22:28:22,578 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(157): Closed 86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:22,578 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=86be86bb450634ab24cca4850cb6c7ec, regionState=CLOSED 2024-11-11T22:28:22,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 because future has completed 2024-11-11T22:28:22,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T22:28:22,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseRegionProcedure 86be86bb450634ab24cca4850cb6c7ec, server=8d82b89dd27d,39613,1731363918313 in 3.1890 sec 2024-11-11T22:28:22,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-11T22:28:22,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86be86bb450634ab24cca4850cb6c7ec, UNASSIGN in 3.1940 sec 2024-11-11T22:28:22,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T22:28:22,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.1980 sec 2024-11-11T22:28:22,589 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731364102589"}]},"ts":"1731364102589"} 2024-11-11T22:28:22,591 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:28:22,591 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:28:22,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.2150 sec 2024-11-11T22:28:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:28:23,543 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:28:23,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:28:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:28:23,545 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:28:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T22:28:23,545 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=160, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:28:23,547 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,549 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C, FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/recovered.edits] 2024-11-11T22:28:23,550 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/0a27c4afc21d4ceabb381895de56f3f2 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/0a27c4afc21d4ceabb381895de56f3f2 2024-11-11T22:28:23,551 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/11009541664b47fdb269a2331b44c19d to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/11009541664b47fdb269a2331b44c19d 2024-11-11T22:28:23,552 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/47bd27ccaf48418db0b748bbaec8f550 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/47bd27ccaf48418db0b748bbaec8f550 2024-11-11T22:28:23,553 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b58e0e51a9d648989220fef086fd441f to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/A/b58e0e51a9d648989220fef086fd441f 2024-11-11T22:28:23,554 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/1e63720cf98e473dade0803c31183ad5 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/1e63720cf98e473dade0803c31183ad5 2024-11-11T22:28:23,555 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/d5f888a767894a8fb5cd03cb1974c3ba to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/d5f888a767894a8fb5cd03cb1974c3ba 2024-11-11T22:28:23,556 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/f01574a66aec409c9259b925ce470b05 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/f01574a66aec409c9259b925ce470b05 2024-11-11T22:28:23,556 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/f183a4736b90407681dc825dce4701f9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/B/f183a4736b90407681dc825dce4701f9 2024-11-11T22:28:23,558 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/2c3ad8143a184bc5926f149b6db0dbe7 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/2c3ad8143a184bc5926f149b6db0dbe7 2024-11-11T22:28:23,559 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/323c06acc5e7446694cc9c241a1e494c to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/323c06acc5e7446694cc9c241a1e494c 2024-11-11T22:28:23,559 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/675fef4253ab4e0eb15e25b9f52f1ff9 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/675fef4253ab4e0eb15e25b9f52f1ff9 2024-11-11T22:28:23,560 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/9c825445a57e4d90b5e1e724d4a98135 to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/C/9c825445a57e4d90b5e1e724d4a98135 2024-11-11T22:28:23,562 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/recovered.edits/303.seqid to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec/recovered.edits/303.seqid 2024-11-11T22:28:23,563 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/default/TestAcidGuarantees/86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,563 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:28:23,563 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:28:23,563 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T22:28:23,565 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111189c868223e24e3aa2d35f458e7452df_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111189c868223e24e3aa2d35f458e7452df_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,566 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115680dd60e7194fbc941cd7224cb3f0bc_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115680dd60e7194fbc941cd7224cb3f0bc_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,567 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111158c640151a324078ab4397290ea43cce_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111158c640151a324078ab4397290ea43cce_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,568 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f39c8384d224aab98fe025c50fe8935_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f39c8384d224aab98fe025c50fe8935_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,568 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111162adae0061c0481e9eb6fabb891680e9_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111162adae0061c0481e9eb6fabb891680e9_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,569 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a707b0890b0741c18f49c03049483d20_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a707b0890b0741c18f49c03049483d20_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,570 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b2484d59d9b144a2a78f947b15951e9a_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b2484d59d9b144a2a78f947b15951e9a_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,571 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b71738bc39574692ae4be0f4d1bec12a_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b71738bc39574692ae4be0f4d1bec12a_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,572 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bb8ffa2c51594bd68a272aaa8b94cd3a_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bb8ffa2c51594bd68a272aaa8b94cd3a_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,573 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c033ce65d32f4c12a10279884fe4569c_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c033ce65d32f4c12a10279884fe4569c_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,573 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c9da996816bd4ed68ad85cc8d2d6853f_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c9da996816bd4ed68ad85cc8d2d6853f_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,574 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d55b5ee5feaf4735ad2575e15132b204_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d55b5ee5feaf4735ad2575e15132b204_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,575 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5ec655a6cc34a9d87bd621e008efb7e_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5ec655a6cc34a9d87bd621e008efb7e_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,575 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d6abdf21a43c4b35b367aa243fdcb6d8_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d6abdf21a43c4b35b367aa243fdcb6d8_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,576 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ddf0ba7fe7a04430a685c6cc75ad226b_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ddf0ba7fe7a04430a685c6cc75ad226b_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,577 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f51da848eba649e78600fc574279964c_86be86bb450634ab24cca4850cb6c7ec to hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f51da848eba649e78600fc574279964c_86be86bb450634ab24cca4850cb6c7ec 2024-11-11T22:28:23,577 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:28:23,579 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=160, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:28:23,581 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:28:23,583 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:28:23,584 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=160, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:28:23,584 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:28:23,584 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731364103584"}]},"ts":"9223372036854775807"} 2024-11-11T22:28:23,585 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:28:23,586 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 86be86bb450634ab24cca4850cb6c7ec, NAME => 'TestAcidGuarantees,,1731364074057.86be86bb450634ab24cca4850cb6c7ec.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:28:23,586 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:28:23,586 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731364103586"}]},"ts":"9223372036854775807"} 2024-11-11T22:28:23,587 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:28:23,588 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=160, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:28:23,589 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-11T22:28:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T22:28:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:28:23,656 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:28:23,662 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239 (was 238) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=367 (was 444), ProcessCount=11 (was 11), AvailableMemoryMB=5381 (was 5431) 2024-11-11T22:28:23,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T22:28:23,663 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:28:23,663 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:28:23,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:23,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:23,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:28:23,663 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T22:28:23,663 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1519452659, stopped=false 2024-11-11T22:28:23,663 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8d82b89dd27d,40133,1731363917560 2024-11-11T22:28:23,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T22:28:23,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T22:28:23,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:28:23,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:28:23,827 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T22:28:23,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T22:28:23,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T22:28:23,828 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:28:23,828 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:28:23,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:23,828 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8d82b89dd27d,39613,1731363918313' ***** 2024-11-11T22:28:23,828 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T22:28:23,829 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T22:28:23,829 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T22:28:23,829 INFO [RS:0;8d82b89dd27d:39613 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T22:28:23,829 INFO [RS:0;8d82b89dd27d:39613 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T22:28:23,829 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(959): stopping server 8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:23,829 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T22:28:23,829 INFO [RS:0;8d82b89dd27d:39613 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8d82b89dd27d:39613. 2024-11-11T22:28:23,829 DEBUG [RS:0;8d82b89dd27d:39613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:28:23,829 DEBUG [RS:0;8d82b89dd27d:39613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:23,830 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T22:28:23,830 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T22:28:23,830 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T22:28:23,830 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T22:28:23,830 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T22:28:23,831 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T22:28:23,831 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:28:23,831 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T22:28:23,831 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T22:28:23,831 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T22:28:23,831 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T22:28:23,831 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T22:28:23,831 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T22:28:23,846 INFO [regionserver/8d82b89dd27d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T22:28:23,855 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/info/23ba760383844b908f432b694ce4a190 is 95, key is TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6./info:/1731363953079/DeleteFamily/seqid=0 2024-11-11T22:28:23,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742430_1606 (size=6514) 2024-11-11T22:28:23,864 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/info/23ba760383844b908f432b694ce4a190 2024-11-11T22:28:23,885 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/ns/b2852fd8423a45408b8afc2c177c711f is 93, key is TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6./ns:/1731363953031/DeleteFamily/seqid=0 2024-11-11T22:28:23,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742431_1607 (size=5906) 2024-11-11T22:28:24,031 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:28:24,231 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:28:24,288 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/ns/b2852fd8423a45408b8afc2c177c711f 2024-11-11T22:28:24,304 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/rep_barrier/f56514780ddc486696b3d12e38cd1f53 is 102, key is TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6./rep_barrier:/1731363953031/DeleteFamily/seqid=0 2024-11-11T22:28:24,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742432_1608 (size=6025) 2024-11-11T22:28:24,431 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:28:24,486 INFO [regionserver/8d82b89dd27d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T22:28:24,486 INFO [regionserver/8d82b89dd27d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T22:28:24,632 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:28:24,707 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/rep_barrier/f56514780ddc486696b3d12e38cd1f53 2024-11-11T22:28:24,723 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/table/2b0f0e26d5e4475a89ab0a8dbf7b6443 is 96, key is TestAcidGuarantees,,1731363923226.765de8aa1edd426d207e09298c569dc6./table:/1731363953031/DeleteFamily/seqid=0 2024-11-11T22:28:24,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742433_1609 (size=5936) 2024-11-11T22:28:24,832 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:28:25,032 DEBUG [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:28:25,127 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/table/2b0f0e26d5e4475a89ab0a8dbf7b6443 2024-11-11T22:28:25,129 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/info/23ba760383844b908f432b694ce4a190 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/info/23ba760383844b908f432b694ce4a190 2024-11-11T22:28:25,132 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/info/23ba760383844b908f432b694ce4a190, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T22:28:25,132 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/ns/b2852fd8423a45408b8afc2c177c711f as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/ns/b2852fd8423a45408b8afc2c177c711f 2024-11-11T22:28:25,135 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/ns/b2852fd8423a45408b8afc2c177c711f, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T22:28:25,136 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/rep_barrier/f56514780ddc486696b3d12e38cd1f53 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/rep_barrier/f56514780ddc486696b3d12e38cd1f53 2024-11-11T22:28:25,138 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/rep_barrier/f56514780ddc486696b3d12e38cd1f53, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T22:28:25,138 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/.tmp/table/2b0f0e26d5e4475a89ab0a8dbf7b6443 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/table/2b0f0e26d5e4475a89ab0a8dbf7b6443 2024-11-11T22:28:25,141 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/table/2b0f0e26d5e4475a89ab0a8dbf7b6443, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T22:28:25,142 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1310ms, sequenceid=90, compaction requested=false 2024-11-11T22:28:25,145 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T22:28:25,145 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T22:28:25,146 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T22:28:25,146 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731364103831Running coprocessor pre-close hooks at 1731364103831Disabling compacts and flushes for region at 1731364103831Disabling writes for close at 1731364103831Obtaining lock to block concurrent updates at 1731364103831Preparing flush snapshotting stores in 1588230740 at 1731364103831Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731364103831Flushing stores of hbase:meta,,1.1588230740 at 1731364103832 (+1 ms)Flushing 1588230740/info: creating writer at 1731364103832Flushing 1588230740/info: appending metadata at 1731364103855 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731364103855Flushing 1588230740/ns: creating writer at 1731364103869 (+14 ms)Flushing 1588230740/ns: appending metadata at 1731364103885 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731364103885Flushing 1588230740/rep_barrier: creating writer at 1731364104291 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731364104304 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731364104304Flushing 1588230740/table: creating writer at 1731364104710 (+406 ms)Flushing 1588230740/table: appending metadata at 1731364104723 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731364104723Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c1eed42: reopening flushed file at 1731364105129 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10d415d5: reopening flushed file at 1731364105132 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@419e5d90: reopening flushed file at 1731364105135 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54db737e: reopening flushed file at 1731364105138 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1310ms, sequenceid=90, compaction requested=false at 1731364105142 (+4 ms)Writing region close event to WAL at 1731364105143 (+1 ms)Running coprocessor post-close hooks at 1731364105145 (+2 ms)Closed at 1731364105146 (+1 ms) 2024-11-11T22:28:25,146 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T22:28:25,232 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(976): stopping server 8d82b89dd27d,39613,1731363918313; all regions closed. 2024-11-11T22:28:25,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741834_1010 (size=25344) 2024-11-11T22:28:25,238 DEBUG [RS:0;8d82b89dd27d:39613 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/oldWALs 2024-11-11T22:28:25,238 INFO [RS:0;8d82b89dd27d:39613 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8d82b89dd27d%2C39613%2C1731363918313.meta:.meta(num 1731363922757) 2024-11-11T22:28:25,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741832_1008 (size=14007479) 2024-11-11T22:28:25,241 DEBUG [RS:0;8d82b89dd27d:39613 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/oldWALs 2024-11-11T22:28:25,241 INFO [RS:0;8d82b89dd27d:39613 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8d82b89dd27d%2C39613%2C1731363918313:(num 1731363921669) 2024-11-11T22:28:25,241 DEBUG [RS:0;8d82b89dd27d:39613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:28:25,241 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T22:28:25,241 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T22:28:25,242 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.ChoreService(370): Chore service for: regionserver/8d82b89dd27d:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T22:28:25,242 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T22:28:25,242 INFO [regionserver/8d82b89dd27d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T22:28:25,242 INFO [RS:0;8d82b89dd27d:39613 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39613 2024-11-11T22:28:25,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8d82b89dd27d,39613,1731363918313 2024-11-11T22:28:25,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T22:28:25,311 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T22:28:25,321 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8d82b89dd27d,39613,1731363918313] 2024-11-11T22:28:25,331 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8d82b89dd27d,39613,1731363918313 already deleted, retry=false 2024-11-11T22:28:25,332 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8d82b89dd27d,39613,1731363918313 expired; onlineServers=0 2024-11-11T22:28:25,332 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8d82b89dd27d,40133,1731363917560' ***** 2024-11-11T22:28:25,332 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T22:28:25,332 INFO [M:0;8d82b89dd27d:40133 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T22:28:25,332 INFO [M:0;8d82b89dd27d:40133 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T22:28:25,332 DEBUG [M:0;8d82b89dd27d:40133 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T22:28:25,332 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T22:28:25,332 DEBUG [M:0;8d82b89dd27d:40133 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T22:28:25,332 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.large.0-1731363921322 {}] cleaner.HFileCleaner(306): Exit Thread[master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.large.0-1731363921322,5,FailOnTimeoutGroup] 2024-11-11T22:28:25,333 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.small.0-1731363921328 {}] cleaner.HFileCleaner(306): Exit Thread[master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.small.0-1731363921328,5,FailOnTimeoutGroup] 2024-11-11T22:28:25,333 INFO [M:0;8d82b89dd27d:40133 {}] hbase.ChoreService(370): Chore service for: master/8d82b89dd27d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T22:28:25,333 INFO [M:0;8d82b89dd27d:40133 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T22:28:25,333 DEBUG [M:0;8d82b89dd27d:40133 {}] master.HMaster(1795): Stopping service threads 2024-11-11T22:28:25,333 INFO [M:0;8d82b89dd27d:40133 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T22:28:25,333 INFO [M:0;8d82b89dd27d:40133 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T22:28:25,333 ERROR [M:0;8d82b89dd27d:40133 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T22:28:25,334 INFO [M:0;8d82b89dd27d:40133 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T22:28:25,334 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T22:28:25,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T22:28:25,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:28:25,346 DEBUG [M:0;8d82b89dd27d:40133 {}] zookeeper.ZKUtil(347): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T22:28:25,346 WARN [M:0;8d82b89dd27d:40133 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T22:28:25,346 INFO [M:0;8d82b89dd27d:40133 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/.lastflushedseqids 2024-11-11T22:28:25,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742434_1610 (size=72) 2024-11-11T22:28:25,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T22:28:25,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1012c00c4dd0001, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T22:28:25,422 INFO [RS:0;8d82b89dd27d:39613 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T22:28:25,422 INFO [RS:0;8d82b89dd27d:39613 {}] regionserver.HRegionServer(1031): Exiting; stopping=8d82b89dd27d,39613,1731363918313; zookeeper connection closed. 2024-11-11T22:28:25,422 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2067e16f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2067e16f 2024-11-11T22:28:25,423 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T22:28:25,757 INFO [M:0;8d82b89dd27d:40133 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T22:28:25,757 INFO [M:0;8d82b89dd27d:40133 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T22:28:25,757 DEBUG [M:0;8d82b89dd27d:40133 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T22:28:25,757 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:28:25,757 DEBUG [M:0;8d82b89dd27d:40133 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:28:25,757 DEBUG [M:0;8d82b89dd27d:40133 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T22:28:25,757 DEBUG [M:0;8d82b89dd27d:40133 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:28:25,757 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=738.68 KB heapSize=907.31 KB 2024-11-11T22:28:25,771 DEBUG [M:0;8d82b89dd27d:40133 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/811df0e392264b68a330a0c4fabb1bb1 is 82, key is hbase:meta,,1/info:regioninfo/1731363922864/Put/seqid=0 2024-11-11T22:28:25,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742435_1611 (size=5672) 2024-11-11T22:28:26,174 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2084 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/811df0e392264b68a330a0c4fabb1bb1 2024-11-11T22:28:26,197 DEBUG [M:0;8d82b89dd27d:40133 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/23123219042d4b5db2b4c94a36e66a75 is 2275, key is \x00\x00\x00\x00\x00\x00\x00%/proc:d/1731363956262/Put/seqid=0 2024-11-11T22:28:26,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742436_1612 (size=42376) 2024-11-11T22:28:26,600 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=738.12 KB at sequenceid=2084 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/23123219042d4b5db2b4c94a36e66a75 2024-11-11T22:28:26,603 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 23123219042d4b5db2b4c94a36e66a75 2024-11-11T22:28:26,617 DEBUG [M:0;8d82b89dd27d:40133 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/809e64e673024ea081c6fb8ab79bbadb is 69, key is 8d82b89dd27d,39613,1731363918313/rs:state/1731363921373/Put/seqid=0 2024-11-11T22:28:26,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073742437_1613 (size=5156) 2024-11-11T22:28:27,021 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2084 (bloomFilter=true), to=hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/809e64e673024ea081c6fb8ab79bbadb 2024-11-11T22:28:27,024 DEBUG [M:0;8d82b89dd27d:40133 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/811df0e392264b68a330a0c4fabb1bb1 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/811df0e392264b68a330a0c4fabb1bb1 2024-11-11T22:28:27,026 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/811df0e392264b68a330a0c4fabb1bb1, entries=8, sequenceid=2084, filesize=5.5 K 2024-11-11T22:28:27,027 DEBUG [M:0;8d82b89dd27d:40133 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/23123219042d4b5db2b4c94a36e66a75 as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/23123219042d4b5db2b4c94a36e66a75 2024-11-11T22:28:27,029 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 23123219042d4b5db2b4c94a36e66a75 2024-11-11T22:28:27,029 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/23123219042d4b5db2b4c94a36e66a75, entries=160, sequenceid=2084, filesize=41.4 K 2024-11-11T22:28:27,030 DEBUG [M:0;8d82b89dd27d:40133 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/809e64e673024ea081c6fb8ab79bbadb as hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/809e64e673024ea081c6fb8ab79bbadb 2024-11-11T22:28:27,032 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46449/user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/809e64e673024ea081c6fb8ab79bbadb, entries=1, sequenceid=2084, filesize=5.0 K 2024-11-11T22:28:27,033 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.HRegion(3140): Finished flush of dataSize ~738.68 KB/756408, heapSize ~907.02 KB/928784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2084, compaction requested=false 2024-11-11T22:28:27,034 INFO [M:0;8d82b89dd27d:40133 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:28:27,034 DEBUG [M:0;8d82b89dd27d:40133 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731364105757Disabling compacts and flushes for region at 1731364105757Disabling writes for close at 1731364105757Obtaining lock to block concurrent updates at 1731364105757Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731364105757Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=756408, getHeapSize=929024, getOffHeapSize=0, getCellsCount=2269 at 1731364105757Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731364105758 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731364105758Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731364105770 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731364105770Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731364106177 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731364106196 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731364106196Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731364106603 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731364106617 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731364106617Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1951e8b9: reopening flushed file at 1731364107024 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d74902a: reopening flushed file at 1731364107026 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a38a3e8: reopening flushed file at 1731364107029 (+3 ms)Finished flush of dataSize ~738.68 KB/756408, heapSize ~907.02 KB/928784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2084, compaction requested=false at 1731364107033 (+4 ms)Writing region close event to WAL at 1731364107034 (+1 ms)Closed at 1731364107034 2024-11-11T22:28:27,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34299 is added to blk_1073741830_1006 (size=893027) 2024-11-11T22:28:27,036 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b6bbe95c-17d1-a565-5339-ec51c6517a2e/MasterData/WALs/8d82b89dd27d,40133,1731363917560/8d82b89dd27d%2C40133%2C1731363917560.1731363920466 not finished, retry = 0 2024-11-11T22:28:27,137 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T22:28:27,137 INFO [M:0;8d82b89dd27d:40133 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T22:28:27,138 INFO [M:0;8d82b89dd27d:40133 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40133 2024-11-11T22:28:27,138 INFO [M:0;8d82b89dd27d:40133 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T22:28:27,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T22:28:27,305 INFO [M:0;8d82b89dd27d:40133 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T22:28:27,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40133-0x1012c00c4dd0000, quorum=127.0.0.1:55789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T22:28:27,310 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T22:28:27,312 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T22:28:27,312 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T22:28:27,312 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T22:28:27,312 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/hadoop.log.dir/,STOPPED} 2024-11-11T22:28:27,315 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T22:28:27,315 WARN [BP-1035355522-172.17.0.3-1731363912033 heartbeating to localhost/127.0.0.1:46449 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T22:28:27,315 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T22:28:27,315 WARN [BP-1035355522-172.17.0.3-1731363912033 heartbeating to localhost/127.0.0.1:46449 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1035355522-172.17.0.3-1731363912033 (Datanode Uuid 5d1d97ad-3b73-4a78-b4d0-810df608b925) service to localhost/127.0.0.1:46449 2024-11-11T22:28:27,318 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/cluster_b777580f-683f-c355-fb6d-abef9c123800/data/data1/current/BP-1035355522-172.17.0.3-1731363912033 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T22:28:27,319 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/cluster_b777580f-683f-c355-fb6d-abef9c123800/data/data2/current/BP-1035355522-172.17.0.3-1731363912033 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T22:28:27,319 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T22:28:27,326 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T22:28:27,327 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T22:28:27,327 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T22:28:27,327 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T22:28:27,327 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f2ff3fda-0506-41b8-be81-e155dc4510af/hadoop.log.dir/,STOPPED} 2024-11-11T22:28:27,341 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T22:28:27,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down